{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RNNCell结构实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNN(nn.Module):\n",
    "    def __init__(self, input_size, hidden_size, output_size):\n",
    "        super(RNN, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.i2h = nn.Linear(input_size + hidden_size, hidden_size)\n",
    "        self.i2o = nn.Linear(input_size + hidden_size, output_size)\n",
    "        self.softmax = nn.LogSoftmax(dim=1)\n",
    "\n",
    "    def forward(self, input, hidden):\n",
    "        combined = torch.cat((input, hidden), 1)#input:[N,26]+[N,hidden_size]=N,26+hidden_size -> N*hidden_size\n",
    "        hidden = self.i2h(combined) #N*hidden_size，这里计算了一个hidden，hidden会带来下一个combined里\n",
    "        output = self.i2o(combined) # N*output_size,就是一个普通全连接层\n",
    "        output = self.softmax(output)#softmax\n",
    "        return output, hidden\n",
    "    def initHidden(self):\n",
    "        return torch.zeros(1, self.hidden_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 9., 15., 24., 23.,  1.,  5., 22., 12., 16., 23., 19.,  4.,  5., 19.,\n",
      "          5.,  5.,  4., 16., 15., 10., 13., 24., 17.,  5., 19.,  7.]])\n",
      "torch.Size([1, 26])\n",
      "torch.Size([1, 2]) torch.Size([1, 10])\n"
     ]
    }
   ],
   "source": [
    "#隐藏层神经元个数\n",
    "n_hidden = 10\n",
    "#分类类别数\n",
    "target_size = 2\n",
    "#输入长度，以字符英文字符为例，英文字符个数为26\n",
    "input_size=26\n",
    "#实例化RNN\n",
    "rnn = RNN(input_size, n_hidden, target_size)\n",
    "#初始化隐藏状态，初始为0\n",
    "hidden = rnn.initHidden()\n",
    "#构造输入\n",
    "input = torch.randint(0,input_size,(1,input_size)).float()\n",
    "print(input)\n",
    "print(input.shape)\n",
    "output, next_hidden = rnn(input, hidden)#得到[1*10,1*128]\n",
    "print(output.data.size(),next_hidden.data.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pytorch的RNN类"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 使用sin预测cos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtUAAAFpCAYAAABTU9T4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X+QXGWd7/HP14GY8voLQlAkg4MXdMVAos4Oog4iUcEfFdAbXVjFcVeLknu53FV0gY2lFpq9yBbrypaai6waLa+gWVnHVWSBwHWqFhkGNyH8kCWwozObLImgxh8bQobv/aNPQ6fT3dM955w+5znP+1U11dOnT8880/P0c779/T7nOebuAgAAALBwTyu6AQAAAEDoCKoBAACAlAiqAQAAgJQIqgEAAICUCKoBAACAlAiqAQAAgJQIqgEAAICUCKoBAACAlAiqAQAAgJQIqgEAAICUDiq6AQtx2GGH+dDQUNHNAAAAQIXdeeedv3D3pd3sG2RQPTQ0pKmpqaKbAQAAgAozs591uy/TPwAAAICUCKoBAACAlAiqAQAAgJSCnFMNAACAYj3++OOanZ3Vnj17im5KaosXL9ayZct08MEHL/hnEFQDAACgZ7Ozs3rWs56loaEhmVnRzVkwd9cjjzyi2dlZHX300Qv+OUz/AAAAQM/27NmjJUuWBB1QS5KZacmSJakz7gTVAAAAWJDQA+q6LP4OgmoAAABUwgc+8AHde++9hfxu5lQDAACgEq6++urCfjeZagAAAPTFzt17dPLlt2jnb9KvGPK73/1Ob33rW7VixQotX75c1157rU455ZQnr7r9zGc+U2vXrtWKFSv0qle9Sg8//HDq39lJJkG1mX3ZzHaa2d1tHjczu9LMtpnZXWb2iobHxszsgeRrLIv2AAAAoHyuvPkBzfzy97ry5m2pf9YPf/hDveAFL9CWLVt099136/TTT9/v8d/97nd61atepS1btujkk0/Wl770pdS/s5OsMtVflXR6h8ffLOnY5OtcSV+UJDM7VNInJJ0oaUTSJ8zskIzaBFRKlp/ugVDQ7xGbKvf5nbv36Nt3zspd2jg1k/pvPP7443XTTTfpoosu0sTEhJ7znOfs9/iiRYv0tre9TZL0yle+UtPT06l+33wyCard/UeSHu2wyxmSvuY1P5b0XDM7QtJpkm5090fd/ZeSblTn4BxYmJlJaeKK2m2gsvx0j0i06/cBvR/a9fsqBx7IAX2+FK68+QE94S5JmnNPfTx78YtfrDvvvFPHH3+8LrnkEl166aX7PX7wwQc/uarHwMCA9u3bl+r3zadfc6qPlDTTcH822dZuO5CdmUlpw2pp07rabeOgGshA2+nTfRUGWuSgXb/v9H4omU79ng+ZaKnVmE6fL4X63/b4XC2ofnzOU2ert2/frmc84xl6z3veo4985CP6yU9+klVzF6RfQXWrxf+8w/YDf4DZuWY2ZWZTu3btyrRxqLjpCWlur+Rztdvpidr2gAbaTp/uQx9okZN2/b7d9hJq1++zLiGjItqN6fT5Umj82+rSZqu3bt2qkZERrVy5UuvWrdPHPvaxtM1MpV9B9aykwYb7yyRt77D9AO5+lbsPu/vw0qVLc2soKmhoVBpYJNlA7XZotLa9pANtc+a506f7Kgy0yEm7ft9ue8l06vdZl5BREe3GdPp8Kdx438NP/m11j8+5brz3Pxb8M0877TTddddd2rx5s+644w4NDw/r1ltv1fDwsCTpt7/97ZP7rlmzRl/96lcX/Lu60a91qsclnW9m16h2UuKv3X2Hmd0g6S8bTk58k6RL+tQmxGJwRBobrw2wQ6O1+9JTA+3c3lINtI2Z50+fubzzp3v3AwbaT5+5vIhmo2za9ft22wu0c/cerVl/mzaed5IOf9ZiSe2zWpdd/1N9/64dBwQeF6w65snnIlLtxvQS9nnpwH5f9T5/+1+8oegm5C6ToNrMvinpFEmHmdmsait6HCxJ7r5e0g8kvUXSNkm/l/QnyWOPmtmnJN2R/KhL3b3TCY/AwgyOHDiQlnCgbc48X7DqmLaf7n949w79Zs++4Ada5KhVv++0vSDNHySl9lmt67fuaPshkw+Ukes0ppesz0sH9nv6fPjMveUU5lIbHh72+sLewH5mJksVJPfqY9dt1bVTM3p8znXwgOmP/vCotoNm47518z0HFZRlny/g/bNz9x6NXn6LHtv3hBYf9DT96KLXd/xQeOJf3qSHdz92wPbnPfvpUWTCkLGCjhm99Psy9/n77rtPL33pSwttQ5Za/T1mdqe7D3fzfC5Tjuqon6RSL/2NjS98kCwouGg1n65d5rnT/DSC6khk3eez+lk9aDVXtFP/LTqIQElkMUYX1Oel3vr9fH2+1fQpFIOgGtXR6iSVhQyQJQgu6joNtgQXyKzPZ/2zutTrB8lufyYBRsVlNUYX0Oel7Pt9q+lTKEa/Vv8A8pfVGd4FrQqS9ZnRrF8dgSxXNShghYQ8lthiickIZDVGF7QqSJb9nhWgyoVMNaojqxMPC1oVJOvMM9mLCGR5sm0BJ+5mPYWp1Ym+ZKsrKKsxuqCT1bPs971On0K+OFERaCXwEx57PfkLqIJeTvRF4AIfo7PQOM7X9Xu850TF/TH9A2hlcEQavTDYwboKFwoAepHHJZBRYoGP0VnIY/pUiL72ta/phBNO0IoVK3TOOefoZz/7mVatWqUTTjhBq1at0s9//nNJ0re//W0tX75cK1as0Mknn5xLWwiqgYLkNeeZ4AIxIsBAbPK4QmFfzExKE1c8dRn5FO655x6tW7dOmzZt0pYtW/S5z31O559/vt773vfqrrvu0rvf/W5dcMEFkqRLL71UN9xwg7Zs2aLx8fHUv7sV5lQDBclrznOvq4gA/ZLnyhwsMYnYBLkCVMara23atElr1qzRYYcdJkk69NBDddttt+k73/mOJOmcc87Rn//5n0uSXvOa1+h973uf3vWud+kd73hH+r+lBYJqoAB5nlBFcIGyyvPk2SADDESBZR4bZLyMobvLzDruU398/fr1uv322/X9739fK1eu1ObNm7VkyZIF/+5WCKqBAuR5xjbBBcqIlTkQK1ZiapDx6lqrVq3S29/+dn3oQx/SkiVL9Oijj+rVr361rrnmGp1zzjn6xje+ode+9rWSpAcffFAnnniiTjzxRH3ve9/TzMxM5kE1c6oRpgznZPUbc54RI06eRXAyOM6wjnST+jKGp67N5MJqL3vZy7R27Vq97nWv04oVK/ThD39YV155pb7yla/ohBNO0Ne//nV97nOfkyR99KMf1fHHH6/ly5fr5JNP1ooVK7L4i/ZDphrhKfDSsllgzjMWpMglxFL+7jyunAjkKqPjDOtItzA4kukYNjY2prGxsf22bdq06YD96vOs80SmGuEp6IqHWQn2jG0Up36A37SudtvPCk0Gv5uVORCcDI4zZahKcmXd/iJTjfAUdMVDSZlkC5nzjJ5lfHJPv393GU6e5WSxQBVVocngOFOGqiTzufuLoBrhKejSsqFPO0HAivwgmcHvLsMHSYKLABU55mZwnCn6wyQnB/cfQTXClPGcrK4UmS3MEBm7ABX1QbLo350RgotAFT3mpjzOFP1hsl/zubtZ1i4E3lRVWAjmVAPdqmfsbKD/2cIMNWbsEJAiL8sc+CWhWXkkUBUZc4vQr/ncixcv1iOPPJJJQFokd9cjjzyixYvTfdgmUw10i4wdEBxWHglYBcbcovRrPveyZcs0OzurXbt2ZfYzi7J48WItW7Ys1c8gqAZ6UcS0kwyxvBNiU4aTxZBC4GNuUfo1n/vggw/W0UcfndnPCx1BNRAJMnaIUdEniwFFKHo+d6wIqoFIkLFDjAguAPQLJyoCOSrTwvtcdAYAgPyQqQZyVKa1ccnYAQCQHzLVQE6aV9ooQ7Ya6IcyVWgAoF8IqoGcsDYuYsVa6ABiRFAN5KBfC+8DZUOFBrGiQgOCapTbzKQ0cUXtNiCdVtoAqowKDSqvzXGJCg0IqlFeM5PShtXSpnW124ACa1baQIyo0KDy2hyXqNBAymj1DzM7XdLnJA1IutrdL2t6/LOSXp/cfYakw939ucljc5K2Jo/93N1XZ9EmVMD0hDS3V/K52u30RDBX1mKlDSzYzGQ4l2VuaitroaPy2hyXuFotpAwy1WY2IOnzkt4s6ThJZ5vZcY37uPuH3H2lu6+U9LeSvtPw8H/WHyOgxn6GRqWBRZIN1G6HRotuEZCvkKozLdpKhQaV1+K4FGKFhvnf+cgiUz0iaZu7PyRJZnaNpDMk3dtm/7MlfSKD34uqGxyRxsbDyNqFlF1EeYVUnWnR1tv/4sKiW4VQhTKGtjguXXnd1uAqNGW6hkKVZDGn+khJMw33Z5NtBzCzF0o6WtKmhs2LzWzKzH5sZme2+yVmdm6y39SuXbsyaDaCMDgijV5Y7kE2pOwiyi2k6kxIbe2AjF0JhDaGNh2XQqvQMP87P1lkqq3FNm+xTZLOkrTR3ecath3l7tvN7EWSNpnZVnd/8IAf6H6VpKskaXh4uN3PB/ovpOwiyi2k6kxIbe2AjF0JBD6GhnYODfO/85NFpnpW0mDD/WWStrfZ9yxJ32zc4O7bk9uHJN0q6eUZtAnoHzJ2yFII1Zm6kNraAhm7kqjIGBqCEOd/hySLoPoOScea2dFmtki1wHm8eScze4mkQyTd1rDtEDN7evL9YZJeo/ZzsYFyqmfsTl1buw00wGCNVcSGNbVLoiJjaAi4hkK+UgfV7r5P0vmSbpB0n6Rvufs9ZnapmTWu5nG2pGvc9/tvvlTSlJltkXSLpMvcnaAa4SFjBwSFjF3JBD6GhiK0+d+hyWSdanf/gaQfNG37eNP9T7Z43j9LOj6LNgBYOObYITasqY0YhTb/OzRcURGIHBk7xIiMHYCsZZKpBhAuMnaIERk7AFkjUw1EjowdAADpkakGIkfGDgCA9MhUAxlgjWcAAOJGUA1kgDWeAQCIG0E1kBJrPCNWVGgA4CkE1UBKXJUNsaJCAwBPIagGUmCNZ8SKCg0A7I+gGkih0xrPQJVRoUGsmPaEdgiqUbyZSWniitptYFjjGTGiQoOYtZ32FPCxDNlgnWoUa2ZS2rBamtsrDSySxsalwZGiW9U11nhGjLgKJ2LVPO3pglXH6PBnLQ7+WIZskKlGsaYnaoOQz9VupyeKbhHQHwFnteat0AT8twGdtJ32xLEMIlONog2N1j7V1z/dD40W3SIgf4FntTpWaAL/24B22k17umDVMTqcYxlEphpFGxypHXRPXcvBF/Goclaryn8botbxxPSKHMs4CTMdgmoUb3BEGr0w2EGoI8rgaKWe1bKB6mW1qvy3Ib2Ax8R5pz1V4FjG2vPpmDd96grB8PCwT01NFd0MoDPK4OhkZrKWxR0arV6/qPLfhoVjTCy1nbv3aPTyW/TYvie0+KCn6UcXvb52EmbkzOxOdx/uZl8y1UBeKIOjkwpktdqqwN9GGTwHjImlxtrz6RFUA3mhDA4EizJ4DhgTS4u157NBUA3khRNXgCBxCfacVGRMrCKuDpwNgmogTxUog5OxQ2wog+eoAmNiFXF14GywTjWAttpePQyoqI5rEdP3UVFcHTgbZKp7RCkcMSFjh9hQBgewUATVPaIUjlhw4gpiRBkcwEIx/aMHlMIRk04Zu0+fubygVgH5ogwOYKHIVPeAUjhiQsYOAIDukanuEievIDZk7AAA6F4mmWozO93M7jezbWZ2cYvH32dmu8xsc/L1gYbHxszsgeRrLIv25IGTVwAAANBO6qDazAYkfV7SmyUdJ+lsMzuuxa7XuvvK5Ovq5LmHSvqEpBMljUj6hJkdkrZNeaAUDonVXwAAQGtZTP8YkbTN3R+SJDO7RtIZku7t4rmnSbrR3R9NnnujpNMlfTODdmWKUjik/Vd/4WQ9AABQl8X0jyMlzTTcn022NftvZnaXmW00s8EenwsUjksXAwCAdrIIqq3FNm+6/z1JQ+5+gqSbJG3o4bm1Hc3ONbMpM5vatWvXghsLLBSrvyBWTHsCgPllEVTPShpsuL9M0vbGHdz9EXd/LLn7JUmv7Pa5DT/jKncfdvfhpUuXZtBsoHtcCAUx46JXADC/LILqOyQda2ZHm9kiSWdJGm/cwcyOaLi7WtJ9yfc3SHqTmR2SnKD4pmQbUCqs/oJYMe0JsaJCg16lDqrdfZ+k81ULhu+T9C13v8fMLjWz1cluF5jZPWa2RdIFkt6XPPdRSZ9SLTC/Q9Kl9ZMWgTJh9RfEimlPiBUVGvTK3FtOYS614eFhn5qaKroZ6NXMpDQ9IQ2NSoMjRbcGwDx27t6j0ctv0WP7nnhy2+KDnqYfXfR6LnqFSmvs+6n7PMe+oJnZne4+3M2+XKYc/TEzKW1YLW1aV7udmSy6RQDmwbQnxCqzCg3HvqgQVKM/piekub2Sz9VupyeKbhHQHzOT0sQVQR5MM5/2FPBrgXhkemI6x76oZHHxF2B+Q6PSwKLaoDKwqHYfqLp6lqre78fGgyr/ZnrRq8BfC8SjU4Wm54t+ceyLCplq9MfgSO0geupaDqaIB1mqp/BaIBCZVmg49kWFTDX6Z3CEAaURJ69UH1mqp/BaxCXg8S3TCo1UiWPfzt17tGb9bdp43kmcpNwBQTVQBErhcahnqQINLjLFaxEPxrfKaVxesOcpMBFh+gdQBErh8RgckUYvJKiQKvFacEGQLjC+VQoXgOoeQTVQhHop3AYohQMB4YIgXWB8qxQuANU9gmqgCJy8AgSHjF2XGN8qI9PlBSNAUA0UhVI4EBQydj2owPgGLgDVK4JqAAtGKRyxIGOHGGV+AaiKY/UPAAvSXAq/YNUxLLWEysr0giBAIDJfXrDiyFRnhDI4YkMpHDEhYwdgPmSqM8IajohJu1I42WpUFRk7APMhU50BzghHbDh5BQCA/RFUZ4AyOGJDKRwAgP0x/SMlyuCIEaVwAAD2R6Y6JcrgAAAAIKhOiTI4AAAAmP6REmXwatq5e4/WrL9NG887iWk8AABgXmSqgRa4UiAAAOgFQTXQhCUSAQBArwiqgSYskYhYcWVYAFg4gmqgQbslEgkyEAOmPQHAwhFUAw1YIhGxYtoTAKRDUA00YIlExIppT4gRU56QJZbUQ/ZmJqXpCWloVBocKbo1PWGJRMSIK8MiVo1Tnj595vL+/vKAj5VojUw1sjUzKW1YLW1aV7udmSy6RQDmwbQnxKjQKU8cKyspk6DazE43s/vNbJuZXdzi8Q+b2b1mdpeZ3WxmL2x4bM7MNidf41m0BwWanpDm9ko+V7udnii6RQDmwbQnxKjQKU8cKysp9fQPMxuQ9HlJb5Q0K+kOMxt393sbdvsXScPu/nszO0/S5ZL+KHnsP919Zdp2oCSGRqWBRbVBYmBR7T4Qg4BLuYVPewr4tUOYCp/yxLGykrKYUz0iaZu7PyRJZnaNpDMkPRlUu/stDfv/WNJ7Mvi9KKPBEWlsnAMk4lIv5dYPkGPj9P1u8dqhAJ2mPPVlbjXHykrKYvrHkZJmGu7PJtvaeb+k6xvuLzazKTP7sZmdmUF7ULTBEWn0QgYJxINS7sLx2qEApZjyxLGycrLIVFuLbd5im8zsPZKGJb2uYfNR7r7dzF4kaZOZbXX3B1s891xJ50rSUUcdlb7VQFlRCg8PpdyF47ULW6DjVeFTnipi5+49WrP+Nm087yRWClI2QfWspMGG+8skbW/eyczeIGmtpNe5+2P17e6+Pbl9yMxulfRySQcE1e5+laSrJGl4eLhl0A4Ej1J4mCjlLhyvXbgYr6JX6JKEJZTF9I87JB1rZkeb2SJJZ0nabxUPM3u5pP8jabW772zYfoiZPT35/jBJr1HDXGwgOpTCw0Upd+F47cLEeBU1rsJ6oNRBtbvvk3S+pBsk3SfpW+5+j5ldamark93+StIzJX27aem8l0qaMrMtkm6RdFnTqiFAXOqlcBugFA4EJMor8zFeRY2rsB7I3MObSTE8POxTU1NFNwPIR6BzFIGYfey6rfrG5M/17hNfGFcZnPEqSjt379Ho5bfosX1PPLlt8UFP048uen3l5lab2Z3uPtzNvlxRESgbSuFAUKIugzNeRYmrsLZGUA0gc1GWwhEtyuCITSmWJCyhLFb/AID9cEY4YlH4lfmAArAkYWtkqgFkKupSOKJDGRxAHUF1ziiDIzaUwhETyuAA6pj+kTPK4IgJpXDEhjI4gDoy1TmiDI7YUAoHAMSKoDpHlMERG0rhAIBYMf0jJ5TBESNK4QCAWJGpzgllcAAAgHgQVOeEMjgAAEA8mP6RE8rgYdi5e4/WrL9NG887iWk5AABgwchUI2qNSx4CAAAsFEE1osWShwAAICsE1YgWSx4iVlzpFQCyR1CNKLVb8pAgAzFg2hMAZI+gGlFiyUPEimlPAJAPgmpEiSUPESumPSFGTHlCP7CkHhZuZlKanpCGRqXBkaJb0xOWPESMuNIrYtU45enTZy4vujmdBXxsjR2ZaizMzKS0YbW0aV3tdmay6BYBmAfTnhCjoKY8cWwNGkE1FmZ6QprbK/lc7XZ6ougWAZgH054Qo6CmPHFsDRrTP7AwQ6PSwKLam35gUe0+EIOAS7NBTnsK+PVG8YKb8sSxNWgE1ViYwRFpbJyDHeJSL83WD3hj4/T9PPF6I6VOU55KObeaY2vQmP6BhRsckUYv5E2PeFCa7S9eb6QU5JQnjq3BIlMNhIIyePEozfYXr3c5BDz2BDnlqQJ27t6jNetv08bzTirnNJucEFQDIaAMXg6UZvuL17t4jD1YgKCWMMwQ0z+AEFAGLw9Ks/3F610sxh70KKglDDNGUA2EoF4GtwHK4EBAgr+SH2MPehTUEoYZyySoNrPTzex+M9tmZhe3ePzpZnZt8vjtZjbU8Nglyfb7zey0LNoDVE69DH7qWsqvQEAay+BBYuxBD9otYRjsh8oepQ6qzWxA0uclvVnScZLONrPjmnZ7v6Rfuvsxkj4r6TPJc4+TdJakl0k6XdIXkp8HoBllcCAolSmDM/agS7FftTWLTPWIpG3u/pC775V0jaQzmvY5Q9KG5PuNklaZmSXbr3H3x9z93yRtS34egAoKvhQO9CDmMjjiFOQShhnKYvWPIyXNNNyflXRiu33cfZ+Z/VrSkmT7j5uee2QGbQJQQrGeEY74BHclPyADsS9hmEWm2lps8y736ea5tR9gdq6ZTZnZ1K5du3psIoCiVaYUDnQh9jI4EKMsgupZSYMN95dJ2t5uHzM7SNJzJD3a5XMlSe5+lbsPu/vw0qVLM2h2sSiDIzaUwhGT2MvgQIyymP5xh6RjzexoSf+u2omHf9y0z7ikMUm3SVojaZO7u5mNS/q/ZvbXkl4g6VhJkxm0qfQogyMmlMIRm9jL4ECMUmeq3X2fpPMl3SDpPknfcvd7zOxSM1ud7PZ3kpaY2TZJH5Z0cfLceyR9S9K9kn4o6X+4+1zaNpUdZXDEhlI4AKDqMlmn2t1/4O4vdvf/6u7rkm0fd/fx5Ps97v5Odz/G3Ufc/aGG565LnvcSd78+i/aUHWVwxIZSOACg6rKY/oEeUAZHjCiFAwCqjsuU9xllcAAAgOohqO4zyuAAAADVw/SPPqMMXoydu/dozfrbtPG8k5hmAwAAMkemGlFoXMIQAAAgawTVqDyWMAQAAHkjqEblsYQhYsWVWwGgfwiqUWntljAkyEAMmPYEAP1DUI1KYwlDxIppTwDQXwTVqDSWMESsmPaEGDHlCUViST1UGksYIkZcuRWxapzy9OkzlxfdHESGTDXmNzMpTVxRuwVQekx7QowqP+WJY3HpEVSjs5lJacNqadO62i1vZqD0mPaEGFV6yhPH4iAw/QOdTU9Ic3sln6vdTk9IgyNFtwpAB0x7QmwqP+WJY3EQyFSjs6FRaWCRZAO126HRolsE9Ael1nLj/4MGlZ/yxLE4CGSq0dngiDQ2XvtUPDTKJ2PEoV5qndtbO4CNjdP3y4T/D5p0mvJUiRMWORYHgaAa8xsc4Q2MuFBqLTf+P2gSxZQnjsWlx/QPIHSUwbNHqbXc+P/kg7EEOav6OuJkqoGQUQbPB6XWcuP/kz3GEvRB1dcRJ1MNhKxVGRzZGByRRi8ksCgr/j/ZYixBziq/jrgIqoGwUQYHglWqUjhjCXJW6XXEEwTVQMjqZfBT11KuBQLTWAovHGMJctRuHfFSfKDMEEE1EDrK4EBwSlkKZyxBTiq/jniCoBpA4UpVBgf6IIZSOFDXaR3xKmH1DwCFq/oZ4UCjyl9SG2gSxTriIlNdOmTsEJtSlsGBHMVSCgdiQ1BdMqU6cQXoA8rgiE0spXAgNqmmf5jZoZKulTQkaVrSu9z9l037rJT0RUnPljQnaZ27X5s89lVJr5P062T397n75jRtCllzxo5SIKqOMjhiFEspHIhN2kz1xZJudvdjJd2c3G/2e0nvdfeXSTpd0t+Y2XMbHv+ou69MvqINqCUydogPZXAAQFWkDarPkLQh+X6DpDObd3D3f3X3B5Lvt0vaKWlpyt9bObGs4Qg0ogwOAKiKtKt/PM/dd0iSu+8ws8M77WxmI5IWSXqwYfM6M/u4kky3uz+Wsk1B6pSxYzUEVBVlcABAVcybqTazm8zs7hZfZ/Tyi8zsCElfl/Qn7v5EsvkSSX8g6Q8lHSrpog7PP9fMpsxsateuXb386iCQsQMAAAiXeVN2tKcnm90v6ZQkS32EpFvd/SUt9nu2pFsl/W93/3abn3WKpI+4+9vm+73Dw8M+NTW14HYDAAAA8zGzO919uJt9086pHpc0lnw/Jum7LRqzSNJ1kr7WHFAngbjMzFSbj313yvYgcqzzDQAAipA2qL5M0hvN7AFJb0zuy8yGzezqZJ93STpZ0vvMbHPytTJ57BtmtlXSVkmHSfp0yvYgcqzzDQAAipBq+kdRmP6BVnbu3qPRy2/RY/ue0OKDnqYfXfR61jpG5e3cvUdr1t+mjeedRH8HgIz1c/oHUBqs840YUZ0BgHIgqEYlsM43YtR8FVb6OwAUh6AalcCV+RAjqjOIFSelo4wIqlEJrPON2FCdQcyY9oQySntFRVTJzKQ0PSENjUqDI0W3pidcmQ+x4SqsiFXztKed2FTIAAAXKUlEQVQLVh0T90m6AR+7q4ZMNWpmJqUNq6VN62q3M5NFtwhAB1RnECumPTXg2F0qZKpRMz0hze2VfK52Oz3BJ16gxKjOIEbtpj1Fm63m2F0qZKpRMzQqDSySbKB2OzRadIuA/M1MShNXkN2pEv6nlcZJ6U04dpcKmWrUDI5IY+PMy0I86mXTub21g9HYOP0+dPxPK6/TtKcozyXg2F0qBNV4yuAIb0jEg7Jp9fA/rTymPbVQgWN3Va4My/QPoKoog3dG2bR6+J92h7EBJVOVJRLJVANVRBl8fpRNq4f/6fwYG1AyVVoikUw1UEWtyuA40OCINHohQUWVVOB/muvVAhkbUDJVWiKRoBqoIsrgQLByLYUzNqBEqnZlWIJqoIrqZfBT11LeBQLSXArPPLhgbECJVG2JRIJqoKoogwPB6UspvAJjA6qhaleG5URFAKXVWAaPcg1aRIWrBSI2VVsikUx1IMjYITa5l8GBkqlaKRyIDUF1IKqyhiPQrSqdEQ50o2qlcCA2TP8IQJXWcAS6QRkcMapaKRyIDZnqAJCxQ2wogwMAQkNQXXJVW8MR6AZlcABAaJj+UXKdMnashoCqogwOAAgNmeqSI2MHAABQfmSqS46MXWs7d+/RmvW3aeN5J3HiGgAAKByZagSJJQYBAECZEFQjOFwUBAAAlA1BNYLDEoOIEVdVBYBySxVUm9mhZnajmT2Q3B7SZr85M9ucfI03bD/azG5Pnn+tmS1K0x5UH0sMIlZMeQKAckubqb5Y0s3ufqykm5P7rfynu69MvlY3bP+MpM8mz/+lpPenbA8qjouCIEZMeUKsqNAgJGmD6jMkbUi+3yDpzG6faGYm6VRJGxfyfMSJJQYRI6Y8IVZUaBCStEvqPc/dd0iSu+8ws8Pb7LfYzKYk7ZN0mbv/g6Qlkn7l7vuSfWYlHZmyPejGzKQ0PSENjUqDI0W3picsMYjYtJvydMGqY1hOEpXWXKGhzy9AwMf7EM0bVJvZTZKe3+KhtT38nqPcfbuZvUjSJjPbKml3i/28xbZ6O86VdK4kHXXUUT38auxnZlLasFqa2ysNLJLGxnmjASXGVVURq1YVGvp8Dzje99280z/c/Q3uvrzF13clPWxmR0hScruzzc/Yntw+JOlWSS+X9AtJzzWzemC/TNL2Du24yt2H3X146dKlPfyJ2M/0RO0N5nO12+mJolsE5G9mUpq4onYbGKY8ZSTgPhAjTkrPAMf7vks7/WNc0piky5Lb7zbvkKwI8nt3f8zMDpP0GkmXu7ub2S2S1ki6pt3zkbGh0don1von16HRolsE5CvwbA1TnjIQeB+IERWaDHC877u0JypeJumNZvaApDcm92Vmw2Z2dbLPSyVNmdkWSbeoNqf63uSxiyR92My2qTbH+u9StgfzGRypHVBOXcuBBXEgWwP6QHCo0GSA433fpcpUu/sjkla12D4l6QPJ9/8s6fg2z39IEv/lfhsc4c0Vs9hOXCFbg1j7QMDvdSo0GQn8eL9z9x6tWX+bNp53UhAnqaad/gEgJDGWwevZmkCDC2Qgxj4Q43sdldO4pGII0364TDkQk1jL4IMj0uiFBBUxq0Af6OlCKLG+11EZIV70iqAaiEm9DG4DcZXBgQro6UIovNcRuBAvekVQDcSEE1eAIPWcteO9joCFuqQiQTUQm9jK4EAFLChrV4H3OuLUaUnFMiOoBhCcnsrgQOBCzdoBCxXqkoqs/hG40JabAdJqLoNfsOoY+j4qjQuhIDahLqlIpjpwZOwQmxBPXgHSCDVrB8SGTHXAyNghNu3K4PR9VFmoWTsgNmSqA0bGDrEJ9eQVAED1EVQHihNXECPK4ACAsmL6R6A4cQUxogwOACgrMtWBiiVjx3rEAAAgBGSqAxVLxq5xdRMy8AAAoKzIVKO0er4sL1ABVGcAIEwE1SgtVjdBjFh7HgDCRFCNUmJ1E8SI6gxiRYUGVUBQjVJiPWLEiOoMYkWFBlVAUF1lM5PSxBW128DEsroJUEd1BrGiQlOAgOODMmP1j6qamZQ2rJbm9koDi6SxcWlwpOhWdS2W1U2AOtaeR6xaVWjo8zkKPD4oMzLVVTU9UXvD+Fztdnqi6BYB+Qs4+0J1piAB95kqoEJTAOKD3JCprqqh0don0Pon0aHRolsE5Cvw7AvVmQIE3meqgApNAYgPckOmuqoGR2oHiFPXcqBAd0LP2JF9Qa+q0mcCfu9SoSkA8UFuyFRX2eAIbxZ0pwoZO7Iv6FUV+kzg710qNAUhPsgFQTWA1hm70AbcevZleqIWHIXWfvRfFfrM9IR8bq/M52q3Ib53gYogqAZQjYydRPYFvQu9zwyN6nEdpKe56wk7SItCfe8CFUBQDaAaGTvVVhJYs/42bTzvJB3+rMVFNwfI3c7nnKD/+fhavdLv0U/sZbryuSfo8KIbBUSKExUB1AyOSKMXBhtQS1yVDfG58uYH9BM/Vl+YO0N3+rH0faBAqYJqMzvUzG40sweS20Na7PN6M9vc8LXHzM5MHvuqmf1bw2Mr07QH+9u5e49OvvwW1vtEFLgqG2LDGs9AuaTNVF8s6WZ3P1bSzcn9/bj7Le6+0t1XSjpV0u8l/VPDLh+tP+7um1O2Bw3I2iEmra7KBlRZpzWeAfRf2qD6DEkbku83SDpznv3XSLre3X+f8vdiHmTtEBMydogRazwD5ZL2RMXnufsOSXL3HWY23/kRZ0n666Zt68zs40oy3e7+WMo2Qa2zdlydClXFVdkQI9Z4Bspl3ky1md1kZne3+Dqjl19kZkdIOl7SDQ2bL5H0B5L+UNKhki7q8PxzzWzKzKZ27drVy6+ODlk7xIaMHQCgaPNmqt297UdhM3vYzI5IstRHSNrZ4Ue9S9J17v54w8/ekXz7mJl9RdJHOrTjKklXSdLw8LC32w/hZe1YBg1pkbEDABQt7ZzqcUljyfdjkr7bYd+zJX2zcUMSiMvMTLX52HenbA8UXtaOEyoRI1bnAYBqSTun+jJJ3zKz90v6uaR3SpKZDUv6oLt/ILk/JGlQ0v9rev43zGypJJO0WdIHU7YHCitr13xC5QWrjiFbjSg0fpgsYwUJANCbVJlqd3/E3Ve5+7HJ7aPJ9ql6QJ3cn3b3I939iabnn+rux7v7cnd/j7v/Nk17EB6WQUOMWJ0HsaJCgyrjioooDCdUIlZ8mESsmO6HKiOoDt3MpDRxRe02MFy4ADHiwyRiRYUmAAHHFGVAUB2ymUlpw2pp07rabWBvgtBOqESJBDzw82EyEAH3sbKiQlNygccUZZD2REUUaXpCmtsr+VztdnpCGhwpulVdC+mESpRIfeCf2ysNLJLGxoPq950+THLCYkkE3sfKqF2FhpPTSyTwmKIMCKpDNjRaG/DrA//QaNEtQhXNTNYG16HRcgywgQ/8fJgMQFn7WNneiz0I7foJUSKmSI2gOmSDI7UMSqCDLAJQxowdAz/yVsY+toD3YpkurEWFJgDEFKkRVPeqbJmCwZFytAPVVMaMHQM/8lbGPraA92KZ1kKnQhOIssUUZYu55kFQ3YsyZu2APJUxYyf1PPCXKWOHQJQtuOjxvciFtRC8AGMuVv/oRatMQYBYfB9dq2fsTl0bxIDWDmvjIng9vhdZaQPBCzDmIqjuRT1TYAPlytr1iAADPRkckUYvDDagZm1cVEaX70XWQkclBBhzEVT3ogJZOwIMxIaMHWLDWuiohABjLoLqXgWetSPAQEzI2CFGXFgLlRFYzMWJihEpevF9ThZDv7E2LmLEShtAMchUR6TokiBzudFvZcjYcWIwAMSBTHVEilx8n+WdUIQyZOzKtFYwACA/BNURKTLAaDWXmwADVceHScSK6X6IEdM/QjEzKU1cUbsNDCeLIVacGIxYMd2vYgKOQfqJoDoE9asKbVpXuw2sUxc9lxsBC3gg58NkRQXcJ/uFpVsrJvAYpJ8IqkMQ4FWFGpXhZDHkJM8AI/CBnA+TFdSvPhl44E6FpmICj0H6iTnVWZmZrHW0odHs11OsX1Vobm8wVxVqVIaTxZCDeoBR75dZL87faiAPZK1SqdgTg5GTfvTJvN9Xyne+c9FLtyIH/YhB8oyh+oigOgt5D4L1qwpVoMOhQvIOMPowkOcZXPBhsoL6EVz0IXDPc0Ua1oavoLxjkD58kOwXpn9koR+lkT5cVYj1dNGTeoBhA/kEGH24RC0nU6En/bhscs7vq7znOzPdr6LyjEEqNL2ETHUWAp+eUcd6uuhJPyoogyO5fZBkuTssSI598smfn+P7Ku/lTanQoGcViaEkMtXZ6Ef2ImdZZi/IeEekDxWUvHAyFUorp/cVK9KglCoQQ9URVGcl4OBCyjbAoKSOsssjuODDJMoujxVp6PfIROAxVB1BNTINMFifFCHII7jgwyTKLo/5zvR74CkE1WVTwPqkWQYYlNSxIH3u91kHF3yYRAhrS9/+F2/Q9GVvPeCrPg+616wz/R4h9Pt+4kTFvPWy9mJBy8rMt55ut8uOsT4pnlTyft/NyVS9LLeX98lfKLmF9uGSrc3b68nq9PvILaTfl6zPZy1VptrM3mlm95jZE2Y23GG/083sfjPbZmYXN2w/2sxuN7MHzOxaM1uUpj2l0+vVtwpaVma+7EW78l5zVoMryEFSMP1+Pt32e07+woL6cMmuGNop69wqg02/R8/9vmR9Pg9pp3/cLekdkn7UbgczG5D0eUlvlnScpLPN7Ljk4c9I+qy7Hyvpl5Len7I95dKpw7UqmeS97u8CdBpom4MO1ieFpOj6PR8m0bEPtyuPl+zDZKepe60+YNLv0XO/L1mfz0Oq6R/ufp8kmVmn3UYkbXP3h5J9r5F0hpndJ+lUSX+c7LdB0iclfTFNm0ql3dqL7UomJbxyYrvyXqs1flmfFJKi6/dcjhxt+3Cn8niJ1ubtNHVPrpbrudPv0XO/L1Gfz0s/5lQfKWmm4f6spBMlLZH0K3ff17D9yHY/xMzOlXSuJB111FH5tDRr7Tpcp8vQ5n1hgR50GmiZS4e2Iuv3fJiEpNZ9eL4+X5IPkx2zzu4tx3r6PST11u9L1OfzMm9QbWY3SXp+i4fWuvt3u/gdrdLY3mF7S+5+laSrJGl4eLjtfqXTqsMF8mmt3UB72fU/1ffv2sEJiWiPfg/M3+dL8mGyXdb5h3fv0G/27KPPozed+n1J+nxe5g2q3T3tx9FZSYMN95dJ2i7pF5Kea2YHJdnq+vbqC+TTWruB9vqtO9pmNchWoy36PWITSJ9vl3X+2HVbde3UzH7b6POYVyD9Pg/mnj7pa2a3SvqIu0+1eOwgSf8qaZWkf5d0h6Q/dvd7zOzbkv7e3a8xs/WS7nL3L8z3+4aHh31q6oBfhT458S9v0sO7Hztg+/Oe/XRKgqgs+j1iQ58HJDO7093brnC3375pgmoze7ukv5W0VNKvJG1299PM7AWSrnb3tyT7vUXS30gakPRld1+XbH+RpGskHSrpXyS9x90PfAc3IagGAABA3voWVBeFoBoAAAB56yWo5jLlAAAAQEoE1QAAAEBKBNUAAABASgTVAAAAQEoE1QAAAEBKBNUAAABASgTVAAAAQEoE1QAAAEBKBNUAAABASgTVAAAAQEoE1QAAAEBK5u5Ft6FnZrZL0s8KbMJhkn5R4O8PCa9Vb3i9esPr1Rter+7xWvWG16s3vF7dK/q1eqG7L+1mxyCD6qKZ2ZS7DxfdjhDwWvWG16s3vF694fXqHq9Vb3i9esPr1b2QXiumfwAAAAApEVQDAAAAKRFUL8xVRTcgILxWveH16g2vV294vbrHa9UbXq/e8Hp1L5jXijnVAAAAQEpkqgEAAICUCKrbMLN3mtk9ZvaEmQ03PXaJmW0zs/vN7LQ2zz/azG43swfM7FozW9Sflhcr+Vs3J1/TZra5zX7TZrY12W+q3+0sCzP7pJn9e8Nr9pY2+52e9LdtZnZxv9tZFmb2V2b2UzO7y8yuM7Pnttkv2v41X18xs6cn79NtyRg11P9WloOZDZrZLWZ2XzLe/68W+5xiZr9ueI9+vIi2lsV87y2ruTLpX3eZ2SuKaGcZmNlLGvrNZjPbbWZ/1rRP1P3LzL5sZjvN7O6GbYea2Y1J/HSjmR3S5rljyT4PmNlY/1rdgbvz1eJL0kslvUTSrZKGG7YfJ2mLpKdLOlrSg5IGWjz/W5LOSr5fL+m8ov+mAl7DKyR9vM1j05IOK7qNRX9J+qSkj8yzz0DSz14kaVHS/44ruu0FvV5vknRQ8v1nJH2mzX5R9q9u+oqk/y5pffL9WZKuLbrdBb5eR0h6RfL9syT9a4vX6xRJ/1h0W8vyNd97S9JbJF0vySS9StLtRbe5DF/Je/M/VFvzuHF71P1L0smSXiHp7oZtl0u6OPn+4lbjvKRDJT2U3B6SfH9I0X8Pmeo23P0+d7+/xUNnSLrG3R9z93+TtE3SSOMOZmaSTpW0Mdm0QdKZeba3bJLX4F2Svll0WypgRNI2d3/I3fdKuka1fhgdd/8nd9+X3P2xpGVFtqeEuukrZ6g2Jkm1MWpV8n6NjrvvcPefJN//RtJ9ko4stlXBO0PS17zmx5Kea2ZHFN2oElgl6UF3L/LCdaXj7j+S9GjT5sYxql38dJqkG939UXf/paQbJZ2eW0O7RFDduyMlzTTcn9WBg/ASSb9qOPi32qfqRiU97O4PtHncJf2Tmd1pZuf2sV1ldH5SJv1ymzJXN30uRn+qWkaslVj7Vzd95cl9kjHq16qNWVFLpsG8XNLtLR4+ycy2mNn1ZvayvjasfOZ7bzFetXaW2ieZ6F/7e56775BqH3wlHd5in1L2s4OKbkCRzOwmSc9v8dBad/9uu6e12Na8hEo3+wSry9ftbHXOUr/G3beb2eGSbjSznyafWCun0+sl6YuSPqVa//iUalNm/rT5R7R4bmX6U7Nu+peZrZW0T9I32vyYaPpXk+jHp4Uws2dK+ntJf+buu5se/olqJfvfJuc8/IOkY/vdxhKZ771F/2qSnFO1WtIlLR6mfy1MKftZ1EG1u79hAU+blTTYcH+ZpO1N+/xCtZLXQUkmqNU+wZrvdTOzgyS9Q9IrO/yM7cntTjO7TrWydSWDnm77mZl9SdI/tniomz5XGV30rzFJb5O0ypPJdS1+RjT9q0k3faW+z2zyXn2ODiy/RsPMDlYtoP6Gu3+n+fHGINvdf2BmXzCzw9z9F/1sZ1l08d6Karzq0psl/cTdH25+gP7V0sNmdoS770imDu1ssc+savPR65apdg5coZj+0btxSWclZ9AfrdonysnGHZID/S2S1iSbxiS1y3xX0Rsk/dTdZ1s9aGb/xcyeVf9etZPP7m61b9U1zTV8u1q/DndIOtZqK8osUq2MON6P9pWNmZ0u6SJJq9399232ibl/ddNXxlUbk6TaGLWp3YeTqkvmkv+dpPvc/a/b7PP8+pxzMxtR7bj5SP9aWR5dvrfGJb03WQXkVZJ+XS/lR6xt5Zb+1VLjGNUufrpB0pvM7JBk2uSbkm3FKvpMybJ+qRbgzEp6TNLDkm5oeGytamfY3y/pzQ3bfyDpBcn3L1It2N4m6duSnl7039TH1+6rkj7YtO0Fkn7Q8NpsSb7uUa2sX3i7C3qtvi5pq6S7VBtIjmh+vZL7b1FtZYIHI3+9tqk2j25z8lVfxYL+1aGvSLpUtQ8ikrQ4GZO2JWPUi4puc4Gv1WtVKxnf1dCn3iLpg/UxTNL5ST/aotrJsa8uut0Fvl4t31tNr5dJ+nzS/7aqYfWsGL8kPUO1IPk5DdvoX0+9Ft+UtEPS40nM9X7VzvG4WdIDye2hyb7Dkq5ueO6fJuPYNkl/UvTf4u5cUREAAABIi+kfAAAAQEoE1QAAAEBKBNUAAABASgTVAAAAQEoE1QAAAEBKBNUAAABASgTVAAAAQEoE1QAAAEBK/x94HNFTPgWLAgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "plt.figure(figsize=(12,6))\n",
    "steps = np.linspace(-10, 10, 100, dtype=np.float32)\n",
    "x_np = np.sin(steps)\n",
    "y_np = np.cos(steps)\n",
    "plt.scatter(steps,x_np,marker=\"^\")\n",
    "plt.scatter(steps,y_np,marker=\".\")\n",
    "plt.legend([\"sin\",\"cos\"])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "bins = 50           # RNN时间步长\n",
    "input_dim = 1       # RNN输入尺寸\n",
    "lr = 0.01           # 初始学习率\n",
    "epochs = 10000       # 轮数\n",
    "hidden_size=64      # 隐藏层神经元个数\n",
    "num_layers = 2      # 神经元层数\n",
    "nonlinearity=\"relu\" #只支持relu和tanh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNNDemo(nn.Module):\n",
    "    def __init__(self,input_dim,hidden_size,num_layers,nonlinearity):\n",
    "        super().__init__()\n",
    "        self.input_dim = input_dim\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        self.nonlinearity = nonlinearity\n",
    "        self.rnn = nn.RNN(\n",
    "            input_size=input_dim,\n",
    "            hidden_size=hidden_size,\n",
    "            num_layers=num_layers,\n",
    "            nonlinearity=nonlinearity\n",
    "        )\n",
    "        self.out = nn.Linear(hidden_size, 1)\n",
    "    def forward(self, x, h):\n",
    "        r_out, h_state = self.rnn(x,h)\n",
    "        outs = [] \n",
    "        for record in range(r_out.size(1)):\n",
    "            outs.append(self.out(r_out[:, record, :]))\n",
    "        return torch.stack(outs, dim=1), h_state\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:0.50047952\n",
      "loss:0.39752430\n",
      "loss:0.41736990\n",
      "loss:0.36897498\n",
      "loss:0.31206673\n",
      "loss:0.78804320\n",
      "loss:0.25809062\n",
      "loss:0.19665539\n",
      "loss:0.14445339\n",
      "loss:0.08041897\n",
      "loss:0.50233680\n",
      "loss:0.48289770\n",
      "loss:0.36406177\n",
      "loss:0.21484376\n",
      "loss:0.22689199\n",
      "loss:0.21282002\n",
      "loss:0.50549072\n",
      "loss:0.50053990\n",
      "loss:0.49695641\n",
      "loss:0.49520889\n",
      "loss:0.49320552\n",
      "loss:0.48983940\n",
      "loss:0.48450744\n",
      "loss:0.46795535\n",
      "loss:0.40585262\n",
      "loss:6.36657667\n",
      "loss:0.38486305\n",
      "loss:0.50129169\n",
      "loss:0.50069618\n",
      "loss:0.50006610\n",
      "loss:0.49957126\n",
      "loss:0.49921104\n",
      "loss:0.49876413\n",
      "loss:0.49800161\n",
      "loss:0.49723935\n",
      "loss:0.49615839\n",
      "loss:0.49461499\n",
      "loss:0.49283111\n",
      "loss:0.48967379\n",
      "loss:0.48405108\n",
      "loss:0.47068140\n",
      "loss:0.42102221\n",
      "loss:0.06667027\n",
      "loss:0.49984738\n",
      "loss:0.50058496\n",
      "loss:0.50037903\n",
      "loss:0.49694893\n",
      "loss:0.48992100\n",
      "loss:0.48677573\n",
      "loss:0.48913416\n",
      "loss:0.50582838\n",
      "loss:0.49045360\n",
      "loss:0.49112642\n",
      "loss:0.47675002\n",
      "loss:0.97081363\n",
      "loss:0.51373130\n",
      "loss:0.49823454\n",
      "loss:0.49846229\n",
      "loss:0.49854097\n",
      "loss:0.49830651\n",
      "loss:0.49823821\n",
      "loss:0.49813244\n",
      "loss:0.49787232\n",
      "loss:0.49733835\n",
      "loss:0.49669978\n",
      "loss:0.49577764\n",
      "loss:0.49468800\n",
      "loss:0.49379760\n",
      "loss:0.49010676\n",
      "loss:0.48236743\n",
      "loss:0.46916252\n",
      "loss:0.44592649\n",
      "loss:0.27712080\n",
      "loss:0.18579893\n",
      "loss:0.09193329\n",
      "loss:0.53031611\n",
      "loss:0.30343536\n",
      "loss:0.21247584\n",
      "loss:0.06650937\n",
      "loss:0.45869103\n",
      "loss:0.51481134\n",
      "loss:0.49910626\n",
      "loss:0.48775077\n",
      "loss:0.50577682\n",
      "loss:0.49233958\n",
      "loss:0.50639963\n",
      "loss:0.47662315\n",
      "loss:0.45252511\n",
      "loss:0.51388550\n",
      "loss:0.51278031\n",
      "loss:0.50273269\n",
      "loss:0.49956203\n",
      "loss:0.49631399\n",
      "loss:0.49538124\n",
      "loss:0.49222329\n",
      "loss:0.48753470\n",
      "loss:0.46846887\n",
      "loss:0.35691205\n",
      "loss:0.51562113\n",
      "loss:0.51929939\n",
      "loss:0.50234640\n",
      "loss:0.50288916\n",
      "loss:0.49824750\n",
      "loss:0.49899366\n",
      "loss:0.49851826\n",
      "loss:0.49678338\n",
      "loss:0.49709752\n",
      "loss:0.49706697\n",
      "loss:0.49742070\n",
      "loss:0.49171498\n",
      "loss:0.57609677\n",
      "loss:0.49824160\n",
      "loss:0.50060117\n",
      "loss:0.50300199\n",
      "loss:0.49610212\n",
      "loss:0.49764785\n",
      "loss:0.49954417\n",
      "loss:0.50118232\n",
      "loss:0.45530578\n",
      "loss:0.23686308\n",
      "loss:0.50158638\n",
      "loss:0.52359051\n",
      "loss:0.50066853\n",
      "loss:0.50234264\n",
      "loss:0.50058508\n",
      "loss:0.49954948\n",
      "loss:0.49954519\n",
      "loss:0.49960282\n",
      "loss:0.49966654\n",
      "loss:0.49958307\n",
      "loss:0.49953327\n",
      "loss:0.49953842\n",
      "loss:0.49953717\n",
      "loss:0.49953020\n",
      "loss:0.49951002\n",
      "loss:0.49950230\n",
      "loss:0.49949390\n",
      "loss:0.49948373\n",
      "loss:0.49947116\n",
      "loss:0.49946201\n",
      "loss:0.49945205\n",
      "loss:0.49943459\n",
      "loss:0.49943182\n",
      "loss:0.49941611\n",
      "loss:0.49941173\n",
      "loss:0.49940175\n",
      "loss:0.49939179\n",
      "loss:0.49938187\n",
      "loss:0.49937189\n",
      "loss:0.49936202\n",
      "loss:0.49934322\n",
      "loss:0.49934205\n",
      "loss:0.49932462\n",
      "loss:0.49932197\n",
      "loss:0.49931186\n",
      "loss:0.49930167\n",
      "loss:0.49929148\n",
      "loss:0.49928114\n",
      "loss:0.49927083\n",
      "loss:0.49925071\n",
      "loss:0.49924973\n",
      "loss:0.49923900\n",
      "loss:0.49922818\n",
      "loss:0.49921724\n",
      "loss:0.49920607\n",
      "loss:0.49919489\n",
      "loss:0.49917406\n",
      "loss:0.49917188\n",
      "loss:0.49915031\n",
      "loss:0.49914813\n",
      "loss:0.49913597\n",
      "loss:0.49912354\n",
      "loss:0.49911097\n",
      "loss:0.49909800\n",
      "loss:0.49908501\n",
      "loss:0.49906322\n",
      "loss:0.49905795\n",
      "loss:0.49903461\n",
      "loss:0.49902961\n",
      "loss:0.49901503\n",
      "loss:0.49899998\n",
      "loss:0.49898475\n",
      "loss:0.49896890\n",
      "loss:0.49895269\n",
      "loss:0.49892914\n",
      "loss:0.49891880\n",
      "loss:0.49889284\n",
      "loss:0.49888256\n",
      "loss:0.49886343\n",
      "loss:0.49884349\n",
      "loss:0.49882317\n",
      "loss:0.49880093\n",
      "loss:0.49877790\n",
      "loss:0.49874926\n",
      "loss:0.49872938\n",
      "loss:0.49870342\n",
      "loss:0.49867427\n",
      "loss:0.49864471\n",
      "loss:0.49861181\n",
      "loss:0.49858293\n",
      "loss:0.49855027\n",
      "loss:0.49851447\n",
      "loss:0.49847469\n",
      "loss:0.49843907\n",
      "loss:0.49840423\n",
      "loss:0.49836838\n",
      "loss:0.49832922\n",
      "loss:0.49828428\n",
      "loss:0.49824739\n",
      "loss:0.49820375\n",
      "loss:0.49815890\n",
      "loss:0.49810973\n",
      "loss:0.49806190\n",
      "loss:0.49800852\n",
      "loss:0.49794474\n",
      "loss:0.49789599\n",
      "loss:0.49783128\n",
      "loss:0.49778077\n",
      "loss:0.49772650\n",
      "loss:0.49768069\n",
      "loss:0.49762663\n",
      "loss:0.49757040\n",
      "loss:0.49750456\n",
      "loss:0.49742734\n",
      "loss:0.49738124\n",
      "loss:0.49734434\n",
      "loss:0.49737683\n",
      "loss:0.49743664\n",
      "loss:0.49749970\n",
      "loss:0.49758053\n",
      "loss:0.49762979\n",
      "loss:0.49741083\n",
      "loss:0.49597758\n",
      "loss:0.48346886\n",
      "loss:0.71677560\n",
      "loss:0.51094031\n",
      "loss:0.49674207\n",
      "loss:0.49966726\n",
      "loss:0.50116158\n",
      "loss:0.49945417\n",
      "loss:0.49418586\n",
      "loss:0.56569344\n",
      "loss:0.51424342\n",
      "loss:0.50550187\n",
      "loss:0.50160348\n",
      "loss:0.50088328\n",
      "loss:0.49542379\n",
      "loss:0.48913887\n",
      "loss:0.45918858\n",
      "loss:0.66621315\n",
      "loss:0.55735505\n",
      "loss:0.46851549\n",
      "loss:0.64455670\n",
      "loss:0.49907768\n",
      "loss:0.49738467\n",
      "loss:0.49587700\n",
      "loss:0.48815840\n",
      "loss:0.47606498\n",
      "loss:0.58214927\n",
      "loss:0.53829038\n",
      "loss:0.49750423\n",
      "loss:0.51040000\n",
      "loss:0.54045069\n",
      "loss:0.46969825\n",
      "loss:0.51641256\n",
      "loss:0.51171488\n",
      "loss:0.50401247\n",
      "loss:0.49546441\n",
      "loss:0.48442999\n",
      "loss:0.46364465\n",
      "loss:0.55066234\n",
      "loss:0.47873324\n",
      "loss:0.52698791\n",
      "loss:0.51290768\n",
      "loss:0.50804222\n",
      "loss:0.50231707\n",
      "loss:0.49782556\n",
      "loss:0.49354348\n",
      "loss:0.49039263\n",
      "loss:0.48836744\n",
      "loss:0.48702657\n",
      "loss:0.48544398\n",
      "loss:0.48323166\n",
      "loss:0.48039481\n",
      "loss:0.47695178\n",
      "loss:0.47235709\n",
      "loss:0.46556005\n",
      "loss:0.45440820\n",
      "loss:0.43257403\n",
      "loss:0.36590719\n",
      "loss:8.39346504\n",
      "loss:0.51336044\n",
      "loss:0.48133734\n",
      "loss:0.48945478\n",
      "loss:0.48626113\n",
      "loss:0.49034807\n",
      "loss:0.48145771\n",
      "loss:0.46296421\n",
      "loss:0.35501370\n",
      "loss:0.20888320\n",
      "loss:0.51083058\n",
      "loss:0.30863112\n",
      "loss:0.60246038\n",
      "loss:0.53831929\n",
      "loss:0.50540382\n",
      "loss:0.49929634\n",
      "loss:0.50019431\n",
      "loss:0.50057310\n",
      "loss:0.50010133\n",
      "loss:0.49969268\n",
      "loss:0.49957949\n",
      "loss:0.49950966\n",
      "loss:0.49941441\n",
      "loss:0.49943134\n",
      "loss:0.49951813\n",
      "loss:0.49946907\n",
      "loss:0.49941680\n",
      "loss:0.49947813\n",
      "loss:0.49950662\n",
      "loss:0.49942252\n",
      "loss:0.49939376\n",
      "loss:0.49947307\n",
      "loss:0.49945205\n",
      "loss:0.49938121\n",
      "loss:0.49939406\n",
      "loss:0.49947518\n",
      "loss:0.49942243\n",
      "loss:0.49937472\n",
      "loss:0.49943998\n",
      "loss:0.49947366\n",
      "loss:0.49939534\n",
      "loss:0.49937278\n",
      "loss:0.49945706\n",
      "loss:0.49944076\n",
      "loss:0.49937403\n",
      "loss:0.49940395\n",
      "loss:0.49946466\n",
      "loss:0.49941108\n",
      "loss:0.49936321\n",
      "loss:0.49942851\n",
      "loss:0.49945420\n",
      "loss:0.49938381\n",
      "loss:0.49937418\n",
      "loss:0.49944550\n",
      "loss:0.49942929\n",
      "loss:0.49935576\n",
      "loss:0.49939331\n",
      "loss:0.49945575\n",
      "loss:0.49940068\n",
      "loss:0.49935281\n",
      "loss:0.49941796\n",
      "loss:0.49944344\n",
      "loss:0.49937281\n",
      "loss:0.49936321\n",
      "loss:0.49943453\n",
      "loss:0.49941832\n",
      "loss:0.49934468\n",
      "loss:0.49938259\n",
      "loss:0.49944487\n",
      "loss:0.49938947\n",
      "loss:0.49934357\n",
      "loss:0.49940619\n",
      "loss:0.49943179\n",
      "loss:0.49934760\n",
      "loss:0.49935153\n",
      "loss:0.49943352\n",
      "loss:0.49940631\n",
      "loss:0.49933204\n",
      "loss:0.49937025\n",
      "loss:0.49943221\n",
      "loss:0.49935845\n",
      "loss:0.49933028\n",
      "loss:0.49939263\n",
      "loss:0.49941763\n",
      "loss:0.49933362\n",
      "loss:0.49933746\n",
      "loss:0.49941933\n",
      "loss:0.49939182\n",
      "loss:0.49931803\n",
      "loss:0.49935547\n",
      "loss:0.49941739\n",
      "loss:0.49934366\n",
      "loss:0.49931523\n",
      "loss:0.49939477\n",
      "loss:0.49940199\n",
      "loss:0.49931768\n",
      "loss:0.49932149\n",
      "loss:0.49940348\n",
      "loss:0.49935752\n",
      "loss:0.49930096\n",
      "loss:0.49935710\n",
      "loss:0.49938995\n",
      "loss:0.49930665\n",
      "loss:0.49927002\n",
      "loss:0.49934363\n",
      "loss:0.49932826\n",
      "loss:0.49925032\n",
      "loss:0.49924627\n",
      "loss:0.49932009\n",
      "loss:0.49926391\n",
      "loss:0.49919769\n",
      "loss:0.49924898\n",
      "loss:0.49928015\n",
      "loss:0.49919617\n",
      "loss:0.49915636\n",
      "loss:0.49922919\n",
      "loss:0.49921295\n",
      "loss:0.49913540\n",
      "loss:0.49915028\n",
      "loss:0.49920389\n",
      "loss:0.49914712\n",
      "loss:0.49908495\n",
      "loss:0.49913564\n",
      "loss:0.49916026\n",
      "loss:0.49908206\n",
      "loss:0.49905106\n",
      "loss:0.49910989\n",
      "loss:0.49909139\n",
      "loss:0.49900246\n",
      "loss:0.49902466\n",
      "loss:0.49907741\n",
      "loss:0.49901956\n",
      "loss:0.49895170\n",
      "loss:0.49900046\n",
      "loss:0.49902153\n",
      "loss:0.49893868\n",
      "loss:0.49890834\n",
      "loss:0.49896550\n",
      "loss:0.49894437\n",
      "loss:0.49885324\n",
      "loss:0.49887493\n",
      "loss:0.49893579\n",
      "loss:0.49888045\n",
      "loss:0.49882379\n",
      "loss:0.49887976\n",
      "loss:0.49890894\n",
      "loss:0.49881950\n",
      "loss:0.49881512\n",
      "loss:0.49889138\n",
      "loss:0.49886203\n",
      "loss:0.49877569\n",
      "loss:0.49880084\n",
      "loss:0.49885896\n",
      "loss:0.49878016\n",
      "loss:0.49873799\n",
      "loss:0.49878958\n",
      "loss:0.49881577\n",
      "loss:0.49872398\n",
      "loss:0.49871841\n",
      "loss:0.49879399\n",
      "loss:0.49876413\n",
      "loss:0.49867773\n",
      "loss:0.49870512\n",
      "loss:0.49876449\n",
      "loss:0.49868616\n",
      "loss:0.49864525\n",
      "loss:0.49871701\n",
      "loss:0.49872521\n",
      "loss:0.49863267\n",
      "loss:0.49862558\n",
      "loss:0.49870023\n",
      "loss:0.49865276\n",
      "loss:0.49858359\n",
      "loss:0.49863055\n",
      "loss:0.49866956\n",
      "loss:0.49859127\n",
      "loss:0.49854943\n",
      "loss:0.49861911\n",
      "loss:0.49862483\n",
      "loss:0.49853116\n",
      "loss:0.49852178\n",
      "loss:0.49859485\n",
      "loss:0.49854645\n",
      "loss:0.49847654\n",
      "loss:0.49852148\n",
      "loss:0.49855754\n",
      "loss:0.49847496\n",
      "loss:0.49843034\n",
      "loss:0.49849755\n",
      "loss:0.49848801\n",
      "loss:0.49840385\n",
      "loss:0.49840862\n",
      "loss:0.49846238\n",
      "loss:0.49841061\n",
      "loss:0.49833840\n",
      "loss:0.49837756\n",
      "loss:0.49840733\n",
      "loss:0.49832764\n",
      "loss:0.49828100\n",
      "loss:0.49834651\n",
      "loss:0.49833557\n",
      "loss:0.49823967\n",
      "loss:0.49825478\n",
      "loss:0.49830788\n",
      "loss:0.49825519\n",
      "loss:0.49818313\n",
      "loss:0.49822110\n",
      "loss:0.49824885\n",
      "loss:0.49816674\n",
      "loss:0.49812645\n",
      "loss:0.49818033\n",
      "loss:0.49816620\n",
      "loss:0.49806717\n",
      "loss:0.49808031\n",
      "loss:0.49813774\n",
      "loss:0.49807760\n",
      "loss:0.49799976\n",
      "loss:0.49803782\n",
      "loss:0.49806198\n",
      "loss:0.49793333\n",
      "loss:0.45111346\n",
      "loss:0.26482803\n",
      "loss:0.14400300\n",
      "loss:0.66067481\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:0.49691859\n",
      "loss:0.49745181\n",
      "loss:0.49858573\n",
      "loss:0.49516729\n",
      "loss:0.49266168\n",
      "loss:0.48166117\n",
      "loss:0.50851989\n",
      "loss:0.50052094\n",
      "loss:0.49851650\n",
      "loss:0.49845320\n",
      "loss:0.49569786\n",
      "loss:0.47289345\n",
      "loss:0.46524784\n",
      "loss:0.61343443\n",
      "loss:0.53336549\n",
      "loss:0.51058894\n",
      "loss:0.49953353\n",
      "loss:0.50299048\n",
      "loss:0.50226015\n",
      "loss:0.50041449\n",
      "loss:0.49984351\n",
      "loss:0.50000334\n",
      "loss:0.50040478\n",
      "loss:0.50114810\n",
      "loss:0.62152404\n",
      "loss:0.53563136\n",
      "loss:0.50912577\n",
      "loss:0.50083923\n",
      "loss:0.49797943\n",
      "loss:0.49439791\n",
      "loss:0.49266213\n",
      "loss:0.48919380\n",
      "loss:0.45276392\n",
      "loss:0.43089336\n",
      "loss:0.36177573\n",
      "loss:0.33545923\n",
      "loss:0.73889196\n",
      "loss:0.57890481\n",
      "loss:0.50396591\n",
      "loss:0.49627841\n",
      "loss:0.49688810\n",
      "loss:0.47620565\n",
      "loss:0.44524342\n",
      "loss:0.47761804\n",
      "loss:0.47860298\n",
      "loss:0.46103689\n",
      "loss:0.38875166\n",
      "loss:0.28881389\n",
      "loss:0.62566566\n",
      "loss:0.49824995\n",
      "loss:0.49757189\n",
      "loss:0.50957495\n",
      "loss:0.51199269\n",
      "loss:0.50419396\n",
      "loss:0.49527326\n",
      "loss:0.48978719\n",
      "loss:0.48732489\n",
      "loss:0.48519504\n",
      "loss:0.48126850\n",
      "loss:0.47684795\n",
      "loss:0.47179374\n",
      "loss:0.46763983\n",
      "loss:0.46185836\n",
      "loss:0.45455077\n",
      "loss:0.44418395\n",
      "loss:0.42929885\n",
      "loss:0.40340006\n",
      "loss:0.33783820\n",
      "loss:0.52639800\n",
      "loss:0.38898009\n",
      "loss:0.38805357\n",
      "loss:0.35547438\n",
      "loss:0.25409940\n",
      "loss:0.16564654\n",
      "loss:0.66572064\n",
      "loss:0.74813807\n",
      "loss:0.54304051\n",
      "loss:0.55341387\n",
      "loss:0.50355649\n",
      "loss:0.50253069\n",
      "loss:0.49663219\n",
      "loss:0.49571657\n",
      "loss:0.49577749\n",
      "loss:0.49542901\n",
      "loss:0.49480766\n",
      "loss:0.49446976\n",
      "loss:0.49424836\n",
      "loss:0.49398214\n",
      "loss:0.49352503\n",
      "loss:0.49307033\n",
      "loss:0.49270409\n",
      "loss:0.49225616\n",
      "loss:0.49162939\n",
      "loss:0.49105006\n",
      "loss:0.49050304\n",
      "loss:0.48976183\n",
      "loss:0.48890471\n",
      "loss:0.48806924\n",
      "loss:0.48714414\n",
      "loss:0.48596266\n",
      "loss:0.48468930\n",
      "loss:0.48336494\n",
      "loss:0.48176241\n",
      "loss:0.47984651\n",
      "loss:0.47769848\n",
      "loss:0.47387451\n",
      "loss:0.46825299\n",
      "loss:0.45512009\n",
      "loss:0.43086338\n",
      "loss:0.42114401\n",
      "loss:0.41953522\n",
      "loss:0.40317842\n",
      "loss:0.29978016\n",
      "loss:0.65427589\n",
      "loss:0.45898610\n",
      "loss:0.46597499\n",
      "loss:0.41940865\n",
      "loss:0.40271643\n",
      "loss:0.36616385\n",
      "loss:0.39695856\n",
      "loss:0.39003062\n",
      "loss:0.36160254\n",
      "loss:0.41769692\n",
      "loss:0.39637187\n",
      "loss:0.22864978\n",
      "loss:0.06860401\n",
      "loss:1.04907656\n",
      "loss:0.58068478\n",
      "loss:0.49966401\n",
      "loss:0.46721759\n",
      "loss:0.46464151\n",
      "loss:0.46964833\n",
      "loss:0.46670565\n",
      "loss:0.46180850\n",
      "loss:0.46021625\n",
      "loss:0.45910758\n",
      "loss:0.45662993\n",
      "loss:0.45390624\n",
      "loss:0.45136291\n",
      "loss:0.44839203\n",
      "loss:0.44490543\n",
      "loss:0.44111463\n",
      "loss:0.43684781\n",
      "loss:0.43177763\n",
      "loss:0.42599621\n",
      "loss:0.41933540\n",
      "loss:0.41133305\n",
      "loss:0.40158430\n",
      "loss:0.38966599\n",
      "loss:0.37477776\n",
      "loss:0.35525507\n",
      "loss:0.32878011\n",
      "loss:0.29123220\n",
      "loss:0.23379245\n",
      "loss:0.14004689\n",
      "loss:0.01767755\n",
      "loss:0.11777804\n",
      "loss:1.83585548\n",
      "loss:0.52293122\n",
      "loss:0.38910800\n",
      "loss:0.43092436\n",
      "loss:0.39403304\n",
      "loss:0.39068407\n",
      "loss:0.38841414\n",
      "loss:0.38478860\n",
      "loss:0.38186818\n",
      "loss:0.37735346\n",
      "loss:0.38380814\n",
      "loss:0.35142136\n",
      "loss:0.32976547\n",
      "loss:0.30783918\n",
      "loss:0.27829748\n",
      "loss:0.22584461\n",
      "loss:0.11804107\n",
      "loss:0.38662174\n",
      "loss:2.45159960\n",
      "loss:0.39389703\n",
      "loss:0.63232142\n",
      "loss:0.37847927\n",
      "loss:0.38774151\n",
      "loss:0.37553486\n",
      "loss:0.37309662\n",
      "loss:0.36039853\n",
      "loss:0.35374951\n",
      "loss:0.34795707\n",
      "loss:0.33920699\n",
      "loss:0.32988060\n",
      "loss:0.32028055\n",
      "loss:0.30914316\n",
      "loss:0.29678860\n",
      "loss:0.28305006\n",
      "loss:0.26731002\n",
      "loss:0.24955015\n",
      "loss:0.22946487\n",
      "loss:0.20650233\n",
      "loss:0.18026786\n",
      "loss:0.15093774\n",
      "loss:0.11848817\n",
      "loss:0.08404981\n",
      "loss:0.05059261\n",
      "loss:0.02286611\n",
      "loss:0.00597851\n",
      "loss:0.00037540\n",
      "loss:0.00013003\n",
      "loss:0.00026815\n",
      "loss:0.00010499\n",
      "loss:0.00001330\n",
      "loss:0.00000425\n",
      "loss:0.00000561\n",
      "loss:0.00000420\n",
      "loss:0.00000376\n",
      "loss:0.00000382\n",
      "loss:0.00000386\n",
      "loss:0.00000418\n",
      "loss:0.00000399\n",
      "loss:0.00000402\n",
      "loss:0.00000398\n",
      "loss:0.00000391\n",
      "loss:0.00000405\n",
      "loss:0.00000409\n",
      "loss:0.00000400\n",
      "loss:0.00000438\n",
      "loss:0.00000437\n",
      "loss:0.00000426\n",
      "loss:0.00000410\n",
      "loss:0.00000404\n",
      "loss:0.00000402\n",
      "loss:0.00000409\n",
      "loss:0.00000407\n",
      "loss:0.00000400\n",
      "loss:0.00000403\n",
      "loss:0.00000404\n",
      "loss:0.00000400\n",
      "loss:0.00000381\n",
      "loss:0.00000380\n",
      "loss:0.00000412\n",
      "loss:0.00000397\n",
      "loss:0.00000402\n",
      "loss:0.00000397\n",
      "loss:0.00000390\n",
      "loss:0.00000405\n",
      "loss:0.00000410\n",
      "loss:0.00000405\n",
      "loss:0.00000437\n",
      "loss:0.00000434\n",
      "loss:0.00000424\n",
      "loss:0.00000411\n",
      "loss:0.00000404\n",
      "loss:0.00000402\n",
      "loss:0.00000410\n",
      "loss:0.00000409\n",
      "loss:0.00000400\n",
      "loss:0.00000407\n",
      "loss:0.00000404\n",
      "loss:0.00000396\n",
      "loss:0.00000380\n",
      "loss:0.00000379\n",
      "loss:0.00000411\n",
      "loss:0.00000396\n",
      "loss:0.00000404\n",
      "loss:0.00000398\n",
      "loss:0.00000393\n",
      "loss:0.00000406\n",
      "loss:0.00000409\n",
      "loss:0.00000435\n",
      "loss:0.00000434\n",
      "loss:0.00000434\n",
      "loss:0.00000427\n",
      "loss:0.00000410\n",
      "loss:0.00000404\n",
      "loss:0.00000403\n",
      "loss:0.00000406\n",
      "loss:0.00000408\n",
      "loss:0.00000400\n",
      "loss:0.00000408\n",
      "loss:0.00000404\n",
      "loss:0.00000392\n",
      "loss:0.00000379\n",
      "loss:0.00000378\n",
      "loss:0.00000403\n",
      "loss:0.00000395\n",
      "loss:0.00000406\n",
      "loss:0.00000397\n",
      "loss:0.00000395\n",
      "loss:0.00000405\n",
      "loss:0.00000409\n",
      "loss:0.00000435\n",
      "loss:0.00000430\n",
      "loss:0.00000432\n",
      "loss:0.00000422\n",
      "loss:0.00000411\n",
      "loss:0.00000403\n",
      "loss:0.00000404\n",
      "loss:0.00000406\n",
      "loss:0.00000406\n",
      "loss:0.00000406\n",
      "loss:0.00000408\n",
      "loss:0.00000418\n",
      "loss:0.00000390\n",
      "loss:0.00000379\n",
      "loss:0.00000381\n",
      "loss:0.00000410\n",
      "loss:0.00000400\n",
      "loss:0.00000405\n",
      "loss:0.00000411\n",
      "loss:0.00000393\n",
      "loss:0.00000407\n",
      "loss:0.00000429\n",
      "loss:0.00000433\n",
      "loss:0.00000435\n",
      "loss:0.00000434\n",
      "loss:0.00000430\n",
      "loss:0.00000423\n",
      "loss:0.00000476\n",
      "loss:0.00000406\n",
      "loss:0.00000452\n",
      "loss:0.00000440\n",
      "loss:0.00000453\n",
      "loss:0.00000418\n",
      "loss:0.00000423\n",
      "loss:0.00000430\n",
      "loss:0.00000668\n",
      "loss:0.00003633\n",
      "loss:0.00044411\n",
      "loss:0.00674857\n",
      "loss:0.07923117\n",
      "loss:0.00244553\n",
      "loss:0.00208791\n",
      "loss:0.00037244\n",
      "loss:0.00566935\n",
      "loss:0.00010244\n",
      "loss:0.00085569\n",
      "loss:0.00017915\n",
      "loss:0.00004691\n",
      "loss:0.00013193\n",
      "loss:0.00011995\n",
      "loss:0.00012800\n",
      "loss:0.00024556\n",
      "loss:0.00092032\n",
      "loss:0.00462777\n",
      "loss:0.02412516\n",
      "loss:0.02168419\n",
      "loss:0.00667965\n",
      "loss:0.00163096\n",
      "loss:0.00096027\n",
      "loss:0.00401157\n",
      "loss:0.00760994\n",
      "loss:0.00505132\n",
      "loss:0.00046738\n",
      "loss:0.00005061\n",
      "loss:0.00057089\n",
      "loss:0.00316819\n",
      "loss:0.01610138\n",
      "loss:0.02765548\n",
      "loss:0.00104163\n",
      "loss:0.00610384\n",
      "loss:0.00076676\n",
      "loss:0.00000959\n",
      "loss:0.00006514\n",
      "loss:0.00043074\n",
      "loss:0.00277715\n",
      "loss:0.02068686\n",
      "loss:0.04271119\n",
      "loss:0.00961875\n",
      "loss:0.00027701\n",
      "loss:0.00579027\n",
      "loss:0.00619921\n",
      "loss:0.00178209\n",
      "loss:0.00001141\n",
      "loss:0.00046166\n",
      "loss:0.00129149\n",
      "loss:0.00386107\n",
      "loss:0.01109262\n",
      "loss:0.01390258\n",
      "loss:0.00110833\n",
      "loss:0.00131562\n",
      "loss:0.00600313\n",
      "loss:0.01967649\n",
      "loss:0.01056098\n",
      "loss:0.00232413\n",
      "loss:0.00616982\n",
      "loss:0.00688672\n",
      "loss:0.00582287\n",
      "loss:0.00230557\n",
      "loss:0.00073402\n",
      "loss:0.00100009\n",
      "loss:0.00474188\n",
      "loss:0.02258562\n",
      "loss:0.01882444\n",
      "loss:0.00489203\n",
      "loss:0.00430415\n",
      "loss:0.00020019\n",
      "loss:0.00009009\n",
      "loss:0.00052421\n",
      "loss:0.00211795\n",
      "loss:0.00912467\n",
      "loss:0.02607151\n",
      "loss:0.00391190\n",
      "loss:0.00559116\n",
      "loss:0.00689959\n",
      "loss:0.00560659\n",
      "loss:0.00311590\n",
      "loss:0.00166627\n",
      "loss:0.00230531\n",
      "loss:0.00818860\n",
      "loss:0.02609285\n",
      "loss:0.00610848\n",
      "loss:0.00538020\n",
      "loss:0.00656982\n",
      "loss:0.00431441\n",
      "loss:0.00206964\n",
      "loss:0.00123081\n",
      "loss:0.00186521\n",
      "loss:0.00583986\n",
      "loss:0.01821363\n",
      "loss:0.01204845\n",
      "loss:0.00105390\n",
      "loss:0.00625178\n",
      "loss:0.01061438\n",
      "loss:0.00747020\n",
      "loss:0.00034120\n",
      "loss:0.00062289\n",
      "loss:0.00374442\n",
      "loss:0.01702582\n",
      "loss:0.02165221\n",
      "loss:0.00128804\n",
      "loss:0.00743000\n",
      "loss:0.00571739\n",
      "loss:0.00335407\n",
      "loss:0.00167731\n",
      "loss:0.00165183\n",
      "loss:0.00417970\n",
      "loss:0.01421260\n",
      "loss:0.01873944\n",
      "loss:0.00013290\n",
      "loss:0.00557972\n",
      "loss:0.00947316\n",
      "loss:0.00780978\n",
      "loss:0.00060976\n",
      "loss:0.00033287\n",
      "loss:0.00221561\n",
      "loss:0.01001973\n",
      "loss:0.02217461\n",
      "loss:0.00160688\n",
      "loss:0.00458112\n",
      "loss:0.01077528\n",
      "loss:0.01313539\n",
      "loss:0.00094375\n",
      "loss:0.00151708\n",
      "loss:0.00662245\n",
      "loss:0.01798258\n",
      "loss:0.00654041\n",
      "loss:0.00166381\n",
      "loss:0.00668108\n",
      "loss:0.01126809\n",
      "loss:0.00459004\n",
      "loss:0.00004801\n",
      "loss:0.00171158\n",
      "loss:0.00641115\n",
      "loss:0.01532210\n",
      "loss:0.00569954\n",
      "loss:0.00053558\n",
      "loss:0.00479623\n",
      "loss:0.01534594\n",
      "loss:0.01193754\n",
      "loss:0.00047538\n",
      "loss:0.00615465\n",
      "loss:0.01426199\n",
      "loss:0.00840963\n",
      "loss:0.00021448\n",
      "loss:0.00410138\n",
      "loss:0.01259110\n",
      "loss:0.01340292\n",
      "loss:0.00006947\n",
      "loss:0.00378529\n",
      "loss:0.01029382\n",
      "loss:0.01115736\n",
      "loss:0.00049680\n",
      "loss:0.00136940\n",
      "loss:0.00551946\n",
      "loss:0.01331029\n",
      "loss:0.00663411\n",
      "loss:0.00011360\n",
      "loss:0.00304363\n",
      "loss:0.01121609\n",
      "loss:0.01635081\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtUAAAF2CAYAAACh02S2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X+U3XV54PH3kxmQbaUEJLEUomGXkQaRxXaI9HhgV0FFrQRX2A2lFV168Gft2e521bCn7tpwSvU02vRoa6rWsItFyTZLbG1pHGCbPRCSSUqFZEiT0qyTBTJpYw2oYXInz/5xv4OTYX7dfO/cn+/XOffce7/3873z0G9vz9NPnu/zRGYiSZIk6eQtaHYAkiRJUrszqZYkSZJKMqmWJEmSSjKpliRJkkoyqZYkSZJKMqmWJEmSSjKpliRJkkoyqZYkSZJKMqmWJEmSSjKpliRJkkrqbXYAJ+Pss8/OpUuXNjsMSZIkdbAdO3b8Q2Yumsvatkyqly5dyuDgYLPDkCRJUgeLiP8717WWf0iSJEklmVRLkiRJJZlUS5IkSSWZVEuSJEklmVRLkiRJJZlUS5IkSSWZVEuSJEklmVRLkiRJJZlUS5IkSSWZVEuSJEkl1SWpjogvR8RIRDw+zecREWsjYl9EfDsifmbCZzdHxN7icXM94pEkSZIaqV471V8Brpnh87cCfcXjVuD3ASLiLOATwOuA5cAnIuLMOsUkSZKkNjVy5ChXfuoBRp492uxQ5qQuSXVm/hVweIYlK4A7s2orsDAizgHeAmzOzMOZ+V1gMzMn55IkSWpDtSbJawf2MvzdH7B2YN88R1YfjaqpPhcYnvD+QHFsuuOSJElqYfOZJI8cOcreHffzgQX3sm9woC12qxuVVMcUx3KG4y/+gohbI2IwIgYPHTpU1+AkSZJUm/lMkjdu2shXelbza7338Ec9q9m46X/VK+x506ik+gCwZML784CnZjj+Ipm5LjP7M7N/0aJF8xaoJElSN6pl53k+k+SRI0d5bs+DnEKF3jjOKVT4/hOtX1vdqKR6E/DuogvI5cD3MvNp4D7gzRFxZnGD4puLY5IkSWqgWnae5zNJXjuwl625jGP0UskFHKOXh3NZy9dW16ul3h8DDwMXRsSBiLglIt4fEe8vlnwTeBLYB/wh8EGAzDwM/CawvXh8sjgmSZKkBqll53m+k+TNQwfZXunjptFVrKncwE2jq9he6WPz7mfq8t86X3rr8SWZeeMsnyfwoWk++zLw5XrEIUmSpGrie/0fPMyGD/wci08/bdb14zvPp1DhGBu5c9PLed9NK6dcu3ZgL3tyGR+kF7LyQpJ8eGAfq6+7+EXrNw8d5GClj5vGVnH5giG2Hl/GzuzjO7ufmXL9I6uuLl69vab/5marS1ItSZKk1jGxlGOqxHWiF3aeF1R3nsnxnefrpkzIuyVJrpVJtSRJUgf5USnHLnYOvpqRqy6Ycbe61p3nbkmSa9WoGxUlSZJ0kmrpzFFrO7p2rWFuNe5US5Iktbi5lnPUWsoB7jzXizvVkiRJLWzkyFHu2XGATNgwODzjbnW7tqPrBCbVkiRJLWztwF4uYQ8f7LmX17BnxgTZUo7msfxDkiSpgWppdzd+0+GdPbcX7e56ee9gTHvzoaUczeNOtSRJUgPVMrlw7cBeLotdJwxa6Y9dlnO0IJNqSZKkBqllciFUyzkeqpxYI/1QZZnlHC3I8g9JkqQGqWVyIYyXc1wNw6+H/VvoXXoFf7JkeeMC1py5Uy1JklTCXHtIv9DubkIpR7Xd3ey9p1myHK74j9VntSSTakmSpBLmWiNtu7vOZlItSZJ0kmqpkbbdXWezplqSJOkk1VIjbbu7zuZOtSRJ0gQNqZFWxzGpliRJmsAaaZ0Mk2pJkqSCNdI6WdZUS5IkFayR1slyp1qSJAlrpFWOSbUkSRLWSKsck2pJktSx5trJA6yRVjnWVEuSpI61cdNGfv7IA2zcdHDa2uhx1kirDJNqSZLUkQ4/sYV37/0Ip/RUOPa3Gzm851zOuvCKZoelDmX5hyRJ6kiDD37jhJsOBx/8RrNDUgczqZYkSR1n5MhRvnTg3BNuOvzy8Ll28tC8MamWJEltY643Hq4d2MvOPPGmwx3ZZycPzRuTakmS1DbmOkJ889BBjo0lO/NVfH5sBTvzVRwbSzt5aN7U5UbFiLgG+F2gB/hiZt4x6fPPAG8o3v4YsDgzFxafjQGPFZ99JzOvrUdMkiSps/xohPgudg6+mpGrLmDx6adNufZHnTykxiidVEdED/A54E3AAWB7RGzKzN3jazLzP0xY/yvAayd8xQ8z89KycUiSpM5WywhxqdHqUf6xHNiXmU9m5ihwN7BihvU3An9ch78rSZK6hCPE1erqkVSfCwxPeH+gOPYiEfFK4Hzg/gmHT4uIwYjYGhHX1SEeSZLUYRwhrlZXj6Q6pjiW06xdCWzIzLEJx16Rmf3ALwCfjYh/MeUfibi1SL4HDx06VC5iSZLUdI4QVyepx42KB4AlE96fBzw1zdqVwIcmHsjMp4rnJyPiQar11n83+cTMXAesA+jv758uaZckSW1iYieP1dddPONaR4ir1dVjp3o70BcR50fEqVQT502TF0XEhcCZwMMTjp0ZES8pXp8NvB7YPflcSZLUWUaOHOWeHQfIhA2Dw9ZGq+2VTqozswJ8GLgPGAK+npm7IuKTETGxPd6NwN2ZOXGXeRkwGBF/AzwA3DGxa4gkSepMawf2cgl7+GDPvbyGPdZGq+3VpU91Zn4T+OakY78x6f1/neK8h4DX1CMGSZLUHsb7Td/Zc3vRHq+X9w7GjH2npVbnREVJktRQawf2clnsOqE9Xn/scrdabc2kWpIkNdTmoYM8VDmxPd5DlWV28lBbq0v5hyRJ0siRo1z/Bw+z4QM/N2MZR7WTx9Uw/HrYv4XepVfwJ0uWNy5QaR64Uy1JkupiYou8OVmyHK74j9Vnqc2ZVEuSpNLGbz78wIJ72Tc4YIs8dR3LPyRJUmkbN23kKz2ri24eG7lz08t5300rmx2W1DDuVEuSpFJGjhzluT0PntDN4/tPzG38uNQpTKolSVIpawf2sjVP7ObxcC6zRZ66iuUfkiSplM1DBzlY6eOmsVVcvmCIrceXsTP7+M7uZ1h93cXNDk9qCJNqSZI0pdpa5AG8vTGBSS3I8g9JkjSlmlvkSV3MpFqSJL2ILfKk2lj+IUmSXsQWeVJt3KmWJEknsEWeVDuTakmSdAJb5Em1s/xDkiSdwBZ5Uu1MqiVJ6hK2yJPmj+UfkiR1CVvkSfPHpFqSpC5gizxpfln+IUlSF7BFnjS/3KmWJKnD2SJPmn8m1ZIkdThb5Enzz/IPSZI6nC3ypPlnUi1JUoezRZ40/yz/kCSpDY0cOcqVn7IuWmoVJtWSJLUhe05LrcWkWpKkNmPPaan1WFMtSVKbsee01HrqslMdEddExJ6I2BcRH5vi8/dExKGIeLR4/PKEz26OiL3F4+Z6xCNJUqey57TUmkon1RHRA3wOeCtwEXBjRFw0xdKvZealxeOLxblnAZ8AXgcsBz4REWeWjUmSpE5lz2mpNdVjp3o5sC8zn8zMUeBuYMUcz30LsDkzD2fmd4HNwDV1iEmSpI60eegg2yt93DS6ijWVG7hpdBXbK31s3v1Ms0OTulo9aqrPBYYnvD9Aded5sndFxJXA3wL/ITOHpzn33DrEJElSR7LntNSa6rFTHVMcy0nvvwEszcxLgG8B62s4t7ow4taIGIyIwUOHDp10sJIktSL7TkvtrR5J9QFgyYT35wFPTVyQmf+Ymc8Xb/8Q+Nm5njvhO9ZlZn9m9i9atKgOYUuS1DrsOy21t3ok1duBvog4PyJOBVYCmyYuiIhzJry9FhgqXt8HvDkizixuUHxzcUySpK5h32mp/ZWuqc7MSkR8mGoy3AN8OTN3RcQngcHM3AR8JCKuBSrAYeA9xbmHI+I3qSbmAJ/MzMNlY5IkqZ3Yd1pqf3UZ/pKZ3wS+OenYb0x4/XHg49Oc+2Xgy/WIQ5KkdvNC3+kF1b7T5Hjf6etYfPppzQ5P0hw5plySpCay77TUGRxTLklSE20eOsjBSh83ja3i8gVDbD2+jJ3Zx3d2P8Pq6y5udniS5sikWpKkJrLvtNQZLP+QJKnO7DktdR+TakmS6sye01L3MamWJKmO7DktdSdrqiVJqiN7TkvdyZ1qSZLq5IWe01R7Tp/CeM9pd6ulTmdSLUlSndhzWupeln9IklQn9pyWupdJtSRJdWLPaal7Wf4hSdIs7DstaTYm1ZIkzcK+05JmY1ItSdIMRo4c5Z4dB8iEDYPD7lZLmpJJtSRJM1g7sJdL2MMHe+7lNexxt1rSlLxRUZKkaYxPR7yz5/ZimEsv7x0MRq66gMWnn9bs8CS1EHeqJUmaxtqBvVwWu04Y5tIfu9ytlvQiJtWSJE1j89BBHqqcOMzlocoyNu9+ptmhSWoxln9IkjSNat/pq2H49bB/C71Lr+BPlixvdliSWpBJtSRJs1myvPqQpGlY/iFJ6joOc5FUbybVkqSu4zAXSfVmUi1J6irjbfI+sOBe9g0OuFstqS6sqZYkdZWNmzbylZ7VRd/pjdy56eW876aVzQ5LUptzp1qS1DVGjhzluT0PntB3+vtPWFstqTyTaklS11g7sJeteWLf6YdzmbXVkkqz/EOS1DU2Dx3kYKWPm8ZWcfmCIbYeX8bO7OM7u59h9XUXNzs8SW3MpFqS1DWqw1wA3t7UOCR1nrqUf0TENRGxJyL2RcTHpvj81yJid0R8OyIGIuKVEz4bi4hHi8emesQjSZIkNVLppDoieoDPAW8FLgJujIiLJi37a6A/My8BNgCfmvDZDzPz0uJxbdl4JEndxUEuklpBPXaqlwP7MvPJzBwF7gZWTFyQmQ9k5g+Kt1uB8+rwdyVJcpCLpJZQj6T6XGB4wvsDxbHp3AL8+YT3p0XEYERsjYjr6hCPJKlLjBw5yj07DpAJGwaH3a2W1DT1SKpjimM55cKIXwT6gU9POPyKzOwHfgH4bET8i2nOvbVIvgcPHTpUNmZJUgdYO7CXS9jDB3vu5TXscbdaUtPUo/vHAWDJhPfnAU9NXhQRVwO3Af8qM58fP56ZTxXPT0bEg8Brgb+bfH5mrgPWAfT390+ZtEuSusf4uPE7e24vpiP28t7BYOSqC1h8+mnNDk9Sl6nHTvV2oC8izo+IU4GVwAldPCLitcAXgGszc2TC8TMj4iXF67OB1wO76xCTJKnDrR3Yy2Wx64TpiP2xy91qSU1ROqnOzArwYeA+YAj4embuiohPRsR4N49PAy8F7pnUOm8ZMBgRfwM8ANyRmSbVkqRZbR46yEOVE6cjPlRZxubdzzQ7NEldKDLbr5Kiv78/BwcHmx2GJKkVDG+D/Vtg6RWwZHmzo5HUQSJiR3Hv36ycqChJam9LlptMS2q6ukxUlCSpXhzmIqkdmVRLklqKw1wktSOTaklSyxhvk/eBBfeyb3DA3WpJbcOaaklSy9i4aSNf6Vld9J3eyJ2bXs77blrZ7LAkaVbuVEuSWsLIkaM8t+fBE/pOf/8Ja6sltQeTaklSS1g7sJeteWLf6YdzmbXVktqC5R+SpJaweeggByt93DS2issXDLH1+DJ2Zh/f2f0Mq6+7uNnhSdKMTKolSS3hkVVXF6/e3tQ4JOlkWP4hSZIklWRSLUmaNw5ykdQtTKolSfPGQS6SuoVJtSRpXjjIRVI38UZFSdK8cJCLpG7iTrUkqe4c5CKp25hUS5LqzkEukrqN5R+SpLpzkIukbmNSLUmqOwe5SOo2ln9IkuZueBts+Z3qsyTpBe5US5LmZngbuf4djFVG6ek9lbj5G7BkebOjkqSW4E61JGlu9m/heGWUXo5zvDIK+7c0OyJJahkm1ZKkOTm8aDmjWe3mMZq9HF7sLrUkjbP8Q5I0J2uGFrJn7DYuy91sj4u4cOhMVl/Y7KgkqTWYVEuSZjVy5Cj37DjA85U+ttMHwGODw3zkqgtYfPppTY5OkprP8g9J0qzWDuzleOYJx8YyHeYiSQWTaknSrDYPHeTY2IlJ9bGxZPPuZ5oUkSS1Fss/JEmz+tEwF0nSVNyplqRu5jAXSaqLuiTVEXFNROyJiH0R8bEpPn9JRHyt+PyRiFg64bOPF8f3RMRb6hGPJGkOhrfB+mvh/turzybWknTSSifVEdEDfA54K3ARcGNEXDRp2S3AdzPzAuAzwG8X514ErAReDVwDfL74PknSfNu/hRwbhRyrPjvMRZJOWj12qpcD+zLzycwcBe4GVkxaswJYX7zeAFwVEVEcvzszn8/Mvwf2Fd8nSZpvS6/gGNVhLsfohaVXNDsiSWpb9bhR8VxgeML7A8DrpluTmZWI+B7wsuL41knnnluHmCRJsxg54xJ+5dht/GzuYme8mrULL2Fxs4OSpDZVj53qmOJYznHNXM6tfkHErRExGBGDhw4dqjFESdJkawf2sjP7+PzYCnZknz2nJamEeiTVB4AlE96fBzw13ZqI6AXOAA7P8VwAMnNdZvZnZv+iRYvqELYkda/xCYnjvaePjSUbBocZefZokyOTpPZUj6R6O9AXEedHxKlUbzzcNGnNJuDm4vX1wP2ZmcXxlUV3kPOBPsDbzyVpnjkhUZLqq3RSnZkV4MPAfcAQ8PXM3BURn4yIa4tlXwJeFhH7gF8DPlacuwv4OrAb+AvgQ5k5VjYmSdLMnJAoSfUVmVOWMLe0/v7+HBwcbHYYkiRJ6mARsSMz++ey1omKkiRJUkkm1ZLUIQ4/sYUv3P4hDu9xiIskNZpJtSR1guFtvPRr7+KW0a/y0rvf5chxSWowk2pJ6gDP7XmABceP0RvHWXD8GM/teaDZIUlSVzGplqQOcNczrzhh5PhdB1/Z7JAkqavUY0y5JKmJRo4cZc0TC7lvbBWXLxhi6/Fl7B46g3c+e5TFp5/W7PAkqSu4Uy1JbW58kMvOfBWfH1vBznyVg1wkqcFMqiWpzTnIRZKaz/IPSWpzj6y6utkhSFLXc6dakiRJKsmkWpJalMNcJKl9mFRLUitymIsktRWTaklqQQ5zkaT2YlItSS3IYS6S1F7s/iFJLcZhLpLUftyplqQW4zAXSWo/JtWS1GIc5iJJ7cfyD0lqMQ5zkaT24061JEmSVJJJtSQ1iMNcJKlzmVRLUiM4zEWSOppJtSQ1gMNcJKmzmVRLUgM4zEWSOpvdPyRpnjnMRZI6nzvVkjTPHOYiSZ3PpFqS5pnDXCSp81n+IUnzzGEuktT5Su1UR8RZEbE5IvYWz2dOsebSiHg4InZFxLcj4t9N+OwrEfH3EfFo8bi0TDySJElSM5Qt//gYMJCZfcBA8X6yHwDvzsxXA9cAn42IhRM+//XMvLR4PFoyHklqCAe5SJImKptUrwDWF6/XA9dNXpCZf5uZe4vXTwEjwKKSf1eSmsdBLpKkScom1S/PzKcBiufFMy2OiOXAqcDfTTh8e1EW8pmIeEnJeCRp3jnIRZI02axJdUR8KyIen+KxopY/FBHnAP8deG9mHi8Ofxz4aeAy4CzgozOcf2tEDEbE4KFDh2r505JUVw5ykSRNNmv3j8yc9rb1iDgYEedk5tNF0jwyzbqfAP4M+C+ZuXXCdz9dvHw+Iv4I+E8zxLEOWAfQ39+f062TpPnkIBdJ0lTKln9sAm4uXt8M3Dt5QUScCmwE7szMeyZ9dk7xHFTrsR8vGY8kzSsHuUiSplI2qb4DeFNE7AXeVLwnIvoj4ovFmn8LXAm8Z4rWeXdFxGPAY8DZwOqS8UjSvHKQiyRpKpHZfpUU/f39OTg42OwwJEmS1MEiYkdm9s9lrWPKJUmSpJJMqiUJh7lIksoxqZYkh7lIkkoyqZbU9RzmIkkqy6RaUtdzmIskqaxZh79IUidzmIskqR7cqZbU1RzmIkmqB5NqSV3NYS6SpHqw/ENSV3tk1dXNDkGS1AHcqZbUmYa3wZbfsT2eJKkh3KmW1HmGt5Hr38FYZZSe3lOJm78BS5Y3OypJUgdzp1pS59m/heOVUXo5zvHKKOx3SqIkaX6ZVEvqOIcXLWc0q32nR7OXw4vdpZYkzS/LPyR1nDVDC9kzdhuX5W62x0VcOHQmqy9sdlSSpE5mUi2po4wcOco9Ow7wfKWP7fQB8NjgMB+56gKHuUiS5o3lH5I6yvgwl4kc5iJJmm8m1ZI6isNcJEnNYPmHpI7iMBdJUjO4Uy2p5R1+YgtfuP1DHN5jazxJUmsyqZbU2oa38dKvvYtbRr/KS+9+lxMSJUktyaRaUkt7bs8DLDh+jN44zoLjx3huzwPNDkmSpBcxqZbU0u565hUcozrI5Ri93HXwlc0OSZKkF/FGRUkta+TIUdY8sZD7xlZx+YIhth5fxu6hM3jns0ftOS1JainuVEtqWeM9p3fmq/j82Ap25qvsOS1Jakkm1ZJalj2nJUntwvIPSS3LntOSpHbhTrWkhrPvtCSp05hUS2os+05LkjpQqaQ6Is6KiM0Rsbd4PnOadWMR8Wjx2DTh+PkR8Uhx/tci4tQy8UhqffadliR1orI71R8DBjKzDxgo3k/lh5l5afG4dsLx3wY+U5z/XeCWkvFIanH2nZYkdaKySfUKYH3xej1w3VxPjIgA3ghsOJnzJbWf8b7TN42uYk3lBm4aXcVnhs5g5NmjzQ5NkqRSyibVL8/MpwGK58XTrDstIgYjYmtEjCfOLwP+KTMrxfsDwLkl45HUwuw7LUnqVLO21IuIbwE/OcVHt9Xwd16RmU9FxD8H7o+Ix4AjU6zLKY6Nx3ErcCvAK17xihr+tKRWMVPf6dXXXdykqCRJKm/WpDozp20UGxEHI+KczHw6Is4BRqb5jqeK5ycj4kHgtcD/BBZGRG+xW30e8NQMcawD1gH09/dPm3xLal32nZYkdaqy5R+bgJuL1zcD905eEBFnRsRLitdnA68HdmdmAg8A1890vqTWZ99pSVK3K5tU3wG8KSL2Am8q3hMR/RHxxWLNMmAwIv6GahJ9R2buLj77KPBrEbGPao31l0rGI6nR7DstSVK5MeWZ+Y/AVVMcHwR+uXj9EPCaac5/ElheJgZJzfXcngc4reg7XSn6Tr90iT9rSVJ3caKipFLsOy1JUsmdakndbbzv9H1jq7h8wRBbjy9j99AZvPPZoyw+/bRmhydJUsO4Uy3ppNl3WpKkKpNqSSdtpr7TkiR1E8s/JJ1oeBvs3wJLr4BZbji077QkSVUm1ZJ+ZHgbuf4djFVG6ek9lbj5G7Mm1pIkyfIPSRPt38Lxyii9HOd4ZbS6Yy1JkmZlUi3pBYcXLWc0q+3xRrOXw4vdpZYkaS4s/5D0gjVDC9kzdhuX5W62x0VcOHQmqy9sdlSSJLU+k2pJQLXn9D07DvB8pY/t9AHw2OAwH7nqAntOS5I0C8s/JAE/6jk9kT2nJUmaG5NqqdMNb4Mtv1N9noE9pyVJOnmWf0idrIYWefacliTp5LlTLXUyW+RJktQQJtVSB7NFniRJjWH5h9TBbJEnSVJjmFRLHcoWeZIkNY7lH1KHskWeJEmNY1IttZM5tscDW+RJktRIln9I7aKG9nhgizxJkhrJnWqpXdgeT5KklmVSLbUJ2+NJktS6LP+Q2oTt8SRJal0m1VIbsD2eJEmtzfIPqZnm2M3D9niSJLU2k2qpWYa3wfpr4f7bq88zJNa2x5MkqbVZ/iE1y/4t5NgokWPV5/1bpm2RZ3s8SZJamzvVUrMsvYJjVLt5HKMXll7R7IgkSdJJKpVUR8RZEbE5IvYWz2dOseYNEfHohMfRiLiu+OwrEfH3Ez67tEw8UjsZOeMSfunYbayp3MC7j93GyMJLmh2SJEk6SWV3qj8GDGRmHzBQvD9BZj6QmZdm5qXAG4EfAH85Ycmvj3+emY+WjEdqG2sH9rIz+/j82Ap2ZJ83HUqS1MbKJtUrgPXF6/XAdbOsvx7488z8Qcm/K7W18RZ54zcfHhtLNgwOM/Ls0SZHJkmSTkbZpPrlmfk0QPG8eJb1K4E/nnTs9oj4dkR8JiJeUjIeqblskSdJUleaNamOiG9FxONTPFbU8oci4hzgNcB9Ew5/HPhp4DLgLOCjM5x/a0QMRsTgoUOHavnTUmMMbyPXv4PKwGpy/TtskSdJUheZtaVeZk7byysiDkbEOZn5dJE0j8zwVf8W2JiZxyZ899PFy+cj4o+A/zRDHOuAdQD9/f053TqpafZv4XhllF6OM1YZpccWeZIkdY2y5R+bgJuL1zcD986w9kYmlX4UiTgREVTrsR8vGY/UNIcXLWc0qy3yRrOXw4unTqglSVLnKTv85Q7g6xFxC/Ad4AaAiOgH3p+Zv1y8XwosAf73pPPviohFQACPAu8vGY/UNGuGFrJn7DYuy91sj4u4cOhMVl/Y7KgkSVIjRGb7VVL09/fn4OBgs8NQNxjeBvu3VAezTFPKAdVuHld86gGerxx/4dhpvQv4q4++gcWnn9aISCVJUp1FxI7M7J/LWicqStOp4cZDu3lIktTdTKql6Uy48fB4ZbS6Yz0Nu3lIktTdytZUSx3r8KLl/LPs5RQqHKOXHy5ezlnTrLWbhyRJ3c2kWt1jjvXR47zxUJIkzZVJtbrD8DZYfy2MjULPqXDzpllvPLxnxwGer/SxnT4AHhsc5iNXXeCNh5Ik6UWsqVZ32L+lmlDnWPV5hvpo8MZDSZJUG5NqdYelV5A9p1BhAdlzSrUEZAbeeChJkmph+Ye6w5LlrFv6Wb73xAOc0fdG3jdLTbU3HkqSpFqYVKt91XDj4ciRo6x5YiHPV1Zw2tAC3vnsUWujJUlS3Vj+ofY0fuPh/bdXn2cYzAIn1khbGy1JkurNpFrtaf8WsrjxMGe58XC8k8d4jfSxsWTD4DAjzx5tVLSSJKnDmVSrPS29gmP0UskFHKN3xhsP7eQhSZLmm0m1WsPwNtjyO7OWcYwbOeMSfunYbayp3MC7j93GyMJLpl1rJw9JkjTfvFFRzVfjYBao7j7vzD4eGbuAU3qCtQP7WH3dxVOutZOHJEmab+5Uq/lqHMxijbQkSWo1JtVqvhoHs1gjLUmSWo1JteZHLTXSxWCWNZUbWHf+785a+mGNtCRSdlzcAAAK2klEQVRJajXWVKv+aqyRrnUwizXSkiSp1bhTrfqroYc0OJhFkiS1P5Nq1V8NPaS96VCSJHUCk2rNTQ010rX0kPamQ0mS1AlMqrtRjYNWGN5Grn8HlYHV5Pp3zHreeA/pz4+tYEf2zZgge9OhJEnqBN6o2AmGt1XrlpdeMWvnjJMZtML+LRyvjNLLccYqo/Ts3zLtOdOVc3zkqgumvPnQmw4lSVIncKe6VrXu8s73948nyfffXn2e7bwaB60AHF60nNGs1kiPZi+HF0+fhFvOIUmSupE71bU4mV3eBuwivyhJnumcYtDKWCXp6TmFmGXQCsCaoYXsGbuNy3I32+MiLhw6k9UXTr12pnKO6caIS5IktTuT6lrUmsDWmiTX+v1QTdZ7Tv3R35gtSS4GrXzviQc4o++NvG+W7x8v53i+0sd2+gB4zHIOSZKkE1j+UYvxBDZ65pbA1lpqUev3AyxZzuHr7+ELvTdy+IZ7Zk3CxwetfL6ygs8MnTFr6zrLOSRJkmZnUl2LJcuru81vvG1upRm1Jsk1Jsjj1gwt5I7n3saaoTNnXVvroBW7c0iSJM2uVPlHRNwA/FdgGbA8MwenWXcN8LtAD/DFzLyjOH4+cDdwFrAT+KXMHC0T07xbsnzOye54knzP/7ybG65fyVlzOG/N0ELueu5tDM9QtzzReHlGJjN22Zi4dq6dOcByDkmSpLkou1P9OPBvgL+abkFE9ACfA94KXATcGBEXFR//NvCZzOwDvgvcUjKeeTdy5ChXfuqBOU/8q2UXeXKCPJe/UcvOs6UckiRJ86NUUp2ZQ5m5Z5Zly4F9mflksQt9N7AiIgJ4I7ChWLceuK5MPI2wdmAvw9/9wZwS0VqT5FpLM2od8W0phyRJ0vxoRPePc4HhCe8PAK8DXgb8U2ZWJhw/twHxnLRaSi1g6iR5urZyJ1OaMdPO81R/x1IOSZKk+THrTnVEfCsiHp/isWKOfyOmOJYzHJ8ujlsjYjAiBg8dOjTHP11ftewk17qLfDKlGe48S5IktYZZd6ozs+z25gFgyYT35wFPAf8ALIyI3mK3evz4dHGsA9YB9Pf3T5t8z5dad5Jr3UU+maEp7jxLkiS1hkaUf2wH+opOH/8PWAn8QmZmRDwAXE+1zvpm4N4GxHNS5jtJNkGWJElqX2Vb6r0T+D1gEfBnEfFoZr4lIn6Kauu8t2VmJSI+DNxHtaXelzNzV/EVHwXujojVwF8DXyoTz3wySZYkSdJ0IrPhlRSl9ff35+DglC2xJUmSpLqIiB2Z2T+XtU5UlCRJkkoyqZYkSZJKMqmWJEmSSjKpliRJkkoyqZYkSZJKMqmWJEmSSjKpliRJkkoyqZYkSZJKMqmWJEmSSjKpliRJkkoyqZYkSZJKisxsdgw1i4hDwP9tdhxt4GzgH5odhLwOLcRr0Rq8Dq3Da9EavA6tYarr8MrMXDSXk9syqdbcRMRgZvY3O45u53VoHV6L1uB1aB1ei9bgdWgNZa+D5R+SJElSSSbVkiRJUkkm1Z1tXbMDEOB1aCVei9bgdWgdXovW4HVoDaWugzXVkiRJUknuVEuSJEklmVS3uYg4LSK2RcTfRMSuiPhvU6x5SUR8LSL2RcQjEbG08ZF2tjleh/dExKGIeLR4/HIzYu0GEdETEX8dEX86xWf+Hhpolmvhb6IBImJ/RDxW/M94cIrPIyLWFr+Jb0fEzzQjzm4wh2vxryPiexN+E7/RjDg7XUQsjIgNEfFERAxFxM9N+vykfhO98xOuGuh54I2Z+VxEnAL8n4j488zcOmHNLcB3M/OCiFgJ/Dbw75oRbAeby3UA+FpmfrgJ8XWbXwWGgJ+Y4jN/D40107UAfxON8obMnK4P8luBvuLxOuD3i2fNj5muBcCWzPz5hkXTnX4X+IvMvD4iTgV+bNLnJ/WbcKe6zWXVc8XbU4rH5EL5FcD64vUG4KqIiAaF2BXmeB3UABFxHvB24IvTLPH30CBzuBZqDSuAO4v/O7YVWBgR5zQ7KGk+RMRPAFcCXwLIzNHM/KdJy07qN2FS3QGKf159FBgBNmfmI5OWnAsMA2RmBfge8LLGRtn55nAdAN5V/FPShohY0uAQu8Vngf8MHJ/mc38PjTPbtQB/E42QwF9GxI6IuHWKz1/4TRQOFMdUf7NdC4CfK0oJ/zwiXt3I4LrEPwcOAX9UlKZ9MSJ+fNKak/pNmFR3gMwcy8xLgfOA5RFx8aQlU+3CuYtaZ3O4Dt8AlmbmJcC3+NFuqeokIn4eGMnMHTMtm+KYv4c6m+O18DfRGK/PzJ+h+k/aH4qIKyd97m+icWa7FjupjsX+l8DvAf+r0QF2gV7gZ4Dfz8zXAt8HPjZpzUn9JkyqO0jxzxcPAtdM+ugAsAQgInqBM4DDDQ2ui0x3HTLzHzPz+eLtHwI/2+DQusHrgWsjYj9wN/DGiPgfk9b4e2iMWa+Fv4nGyMyniucRYCOwfNKSF34ThfOApxoTXXeZ7Vpk5pHxUsLM/CZwSkSc3fBAO9sB4MCEf03eQDXJnrym5t+ESXWbi4hFEbGweP3PgKuBJyYt2wTcXLy+Hrg/bVBeV3O5DpPqsa6levOW6igzP56Z52XmUmAl1f9d/8VJy/w9NMBcroW/ifkXET8eEaePvwbeDDw+adkm4N1Fx4PLge9l5tMNDrXjzeVaRMRPjt/jERHLqeZp/9joWDtZZj4DDEfEhcWhq4Ddk5ad1G/C7h/t7xxgfUT0UP3xfT0z/zQiPgkMZuYmqsX4/z0i9lHdkVvZvHA71lyuw0ci4lqgQvU6vKdp0XYZfw+tw99Ew70c2Fjkab3AVzPzLyLi/QCZ+QfAN4G3AfuAHwDvbVKsnW4u1+J64AMRUQF+CKz0/+mfF78C3FV0/ngSeG89fhNOVJQkSZJKsvxDkiRJKsmkWpIkSSrJpFqSJEkqyaRakiRJKsmkWpIkSQ0XEb9ZTFR9NCL+MiJ+app1fxER/xQRfzrN578XEc9NeH9lROyMiEpEXF9DPD3FlMUp/85sTKolSZI0ryLiX0fEVyYd/nRmXlJMI/5T4DemOf3TwC9N8739wMJJh79DtUXnV2sM81cp0S/fpFqSJEkNl5lHJrz9caYZBZ6ZA8Czk48XsyE+DfznSev3Z+a3geNTnPPrEbG92CH/bxOOnwe8Hfjiyfy3gMNfJEmS1CQRcTvwbuB7wBtqPP3DwKbMfLoYqjPb33oz0Ed1PHwAmyLiysz8K+CzVJPz02uM4QXuVEuSJGleRMQjEfEo1R3ga4v66Ucj4i0AmXlbZi4B7qKaJM/1e38KuAH4vRrCeXPx+GtgJ/DTQF9E/Dwwkpk7aviuF3GnWpIkSfMiM18H1Zpq4D2Z+Z5pln4V+DPgE3P86tcCFwD7il3qH4uIfZl5wQznBPBbmfmFEw5G/BbVhP9twGnAT0TE/8jMX5xjLIA71ZIkSWqCiOib8PZa4Im5npuZf5aZP5mZSzNzKfCDWRJqgPuAfx8RLy3+/rkRsTgzP56Z5xXfsxK4v9aEGkyqJUmS1Bx3RMTjEfFtqmUZvwrVjh4R8cINgxGxBbgHuCoiDoyXjkwnIi6LiANUy0O+EBG7ADLzL6nuiD8cEY8BGyhRQ/2iv5s55Y2WkiRJkubInWpJkiSpJJNqSZIkqSSTakmSJKkkk2pJkiSpJJNqSZIkqSSTakmSJKkkk2pJkiSpJJNqSZIkqaT/D1U0U6qbsqq2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 864x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "rnnDemo = RNNDemo(input_dim,hidden_size,num_layers,nonlinearity)\n",
    "optimizer = torch.optim.Adam(rnnDemo.parameters(), lr=lr)\n",
    "loss_func = nn.MSELoss()\n",
    "\n",
    "h_state = None\n",
    "for step in range(epochs):\n",
    "    start, end = step * np.pi, (step + 1) * np.pi  # 时间跨度\n",
    "    # 使用Sin函数预测Cos函数\n",
    "    steps = np.linspace(start, end, bins, dtype=np.float32, endpoint=False)\n",
    "    x_np = np.sin(steps)\n",
    "    y_np = np.cos(steps)\n",
    "    x = torch.from_numpy(x_np).unsqueeze(1).unsqueeze(2)#【100，1,1】尺寸大小为(time_step, batch, input_size)\n",
    "    y = torch.from_numpy(y_np).unsqueeze(1).unsqueeze(2)#【100，1,1】\n",
    "    prediction, h_state = rnnDemo(x, h_state)  # RNN输出（预测结果，隐藏状态）\n",
    "    #将每一次输出的中间状态传递下去(不带梯度)\n",
    "    h_state = h_state.detach()  \n",
    "    loss = loss_func(prediction, y)\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    if(step%10==0):\n",
    "        print(\"loss:{:.8f}\".format(loss))\n",
    "\n",
    "plt.figure(figsize=(12,6))\n",
    "plt.scatter(steps,y_np,marker=\"^\")\n",
    "plt.scatter(steps, prediction.data.numpy().flatten(),marker=\".\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LstmDemo(nn.Module):\n",
    "    def __init__(self,input_dim,hidden_size,num_layers):\n",
    "        super().__init__()\n",
    "        self.input_dim = input_dim\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        self.lstm = nn.LSTM(\n",
    "            input_size=input_dim,\n",
    "            hidden_size=hidden_size,\n",
    "            num_layers=num_layers\n",
    "        )\n",
    "        self.out = nn.Linear(hidden_size, 1)\n",
    "\n",
    "    def forward(self, x, h_0_c_0):\n",
    "        r_out, h_state = self.lstm(x,h_0_c_0)\n",
    "        outs = [] \n",
    "        #r_out:(h_n, c_n)\n",
    "        for record in range(r_out.size(1)):\n",
    "            outs.append(self.out(r_out[:, record, :]))\n",
    "        return torch.stack(outs, dim=1), h_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:0.51717800\n",
      "loss:0.50065255\n",
      "loss:0.49995759\n",
      "loss:0.47315955\n",
      "loss:0.50064230\n",
      "loss:0.57572514\n",
      "loss:0.01025884\n",
      "loss:0.00062259\n",
      "loss:0.00181259\n",
      "loss:0.00138456\n",
      "loss:0.00026689\n",
      "loss:0.00006099\n",
      "loss:0.00056895\n",
      "loss:0.00005902\n",
      "loss:0.00075268\n",
      "loss:0.00029600\n",
      "loss:0.00080693\n",
      "loss:0.00045928\n",
      "loss:0.00088534\n",
      "loss:0.00083844\n",
      "loss:0.00012703\n",
      "loss:0.00015219\n",
      "loss:0.00076467\n",
      "loss:0.00117184\n",
      "loss:0.00037389\n",
      "loss:0.00045633\n",
      "loss:0.00069958\n",
      "loss:0.00003411\n",
      "loss:0.00031411\n",
      "loss:0.00015287\n",
      "loss:0.00020209\n",
      "loss:0.00045334\n",
      "loss:0.00065083\n",
      "loss:0.00003659\n",
      "loss:0.00067867\n",
      "loss:0.00016141\n",
      "loss:0.00075547\n",
      "loss:0.00034406\n",
      "loss:0.00066487\n",
      "loss:0.00051539\n",
      "loss:0.00048299\n",
      "loss:0.00074988\n",
      "loss:0.00012824\n",
      "loss:0.00082202\n",
      "loss:0.00072793\n",
      "loss:0.00000855\n",
      "loss:0.00003252\n",
      "loss:0.00079097\n",
      "loss:0.00059037\n",
      "loss:0.00058158\n",
      "loss:0.00071064\n",
      "loss:0.00061476\n",
      "loss:0.00027594\n",
      "loss:0.00019050\n",
      "loss:0.00062342\n",
      "loss:0.00073978\n",
      "loss:0.00061647\n",
      "loss:0.00011231\n",
      "loss:0.00000999\n",
      "loss:0.00001152\n",
      "loss:0.00065350\n",
      "loss:0.00020106\n",
      "loss:0.00075714\n",
      "loss:0.00039635\n",
      "loss:0.00086366\n",
      "loss:0.00004017\n",
      "loss:0.00141748\n",
      "loss:0.00012015\n",
      "loss:0.00000164\n",
      "loss:0.00000083\n",
      "loss:0.00000782\n",
      "loss:0.00346634\n",
      "loss:0.00000910\n",
      "loss:0.00000168\n",
      "loss:0.00000149\n",
      "loss:0.00002510\n",
      "loss:0.00011938\n",
      "loss:0.00023499\n",
      "loss:0.00000474\n",
      "loss:0.00000092\n",
      "loss:0.00000106\n",
      "loss:0.00000068\n",
      "loss:0.00213265\n",
      "loss:0.00002371\n",
      "loss:0.00000141\n",
      "loss:0.00000126\n",
      "loss:0.00000310\n",
      "loss:0.00174980\n",
      "loss:0.00004674\n",
      "loss:0.00000245\n",
      "loss:0.00000107\n",
      "loss:0.00021004\n",
      "loss:0.00085779\n",
      "loss:0.00000098\n",
      "loss:0.00000566\n",
      "loss:0.00004895\n",
      "loss:0.00098197\n",
      "loss:0.00000193\n",
      "loss:0.00000131\n",
      "loss:0.00000046\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEJCAYAAAB/pOvWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X+UXGWd5/H3J90JQScMIAkgNIYxWQwkLDBFwMPBM5Dwc2aTzMqM4LgEl5ysIotHdoQInnGGCQj6By6uq0ZEw4qC5MAQFcUQYGEPBKhoJCQxpMWMiQHSCphhMIFOf/ePut2pVN3bXdVV3fWjP69z6lTd5z6367lcTn3zPN/nPlcRgZmZWbFxjW6AmZk1HwcHMzMr4+BgZmZlHBzMzKyMg4OZmZVxcDAzszIODmZmVsbBwczMyjg4mJlZmc5GN2A4DjvssJg6dWqjm2Fm1lLWrl37u4iYXEndlgwOU6dOJZ/PN7oZZmYtRdK/VlrXw0pmZlbGwcHMzMo4OJiZWRkHBzMzK+PgYGZmZRwczMysTF2Cg6Q7JO2U9HzGfkm6TVK3pOcknVK0b6GkLclrYT3aY2ZmtalXz+HbwPmD7L8AmJ68FgNfBZB0KPA54DRgNvA5SYfUqU1mZi1l567dfOALj7Lz33ZXVD6S6hIcIuJx4NVBqswH7oyCNcDBko4EzgNWRcSrEfEasIrBg4yZWcvL+rG/bfUWtr32Jret7q6ofCSNVs7hKGBb0fb2pCyrvIykxZLykvI9PT0j1lAzs3qpJgjs3LWbe9duJwJW5LcNHJNVPtJGKzgopSwGKS8vjFgWEbmIyE2eXNHSIGZmDVVNELht9RZOZDNXdDzALDYPHJNVPtJGa22l7UBX0fbRwI6k/C9Kyh8bpTaZmdVs567dXPS1p1jx8fczZdLE/cqLg8BVc6YxZdLEgR/7Uzs28izHc9vqLq46expb1j7CnR03Mp5e3qaTj+bFxtldqeU7k781kkar57ASuDSZtXQ68IeIeAl4CDhX0iFJIvrcpMzMrCUMlifoi8JAyN4Iblvdzc5duwd+7K/uvJc7O26kO7+aW37yS07VBsbTS6f6GE8vOW3gk3evSy0fjd5Dvaayfg94CjhO0nZJl0v6mKSPJVUeBF4EuoFvAFcARMSrwD8DzyavG5IyM7OmkpY/GCpPMKsvGQ7q28yK/LbMIPDg+pd5sncGb9NJb4zjbTp5sncGv+p5I7V81caXR/x86zKsFBGXDLE/gE9k7LsDuKMe7TAzGynFPYSlC2YOlJUOES1dMHOg/M4JNw0MB12693oeXC9m9M7gExM6IXoHfuwPmtTJfdd9CradAVufoHPqmdzXNTv55r/MKB9ZLfk8BzOz0ZSWPyDIzAes2vQKH4yNAz0EopdTYyO/ecfMQYIA0DW78CqVVT6CHBzMzIqkJZjTeghE7DdEROzLBzx93VzYdhAsXwl736KzcwLXLFrENf0/8A34sa+Wg4OZWZHS4aPiJHJxD+FXB55AV8oQ0W83vlwYduqaDQtXwtYnYOqZTR8MSjk4mNmYlNZDSBs+um31ltQewrQT5rB0wdzB8wEt0EPI4lVZzWxMSpuCmnbD2apNrww+Y6hrNpz5P1o2CGRxz8HMxpxqEsw/uPa/MWXSED2ENuSeg5m1tbT7E9JuUCsdPiq74axNewhZHBzMrK2VDh9l3aD20MYhho/GGA8rmVnbykowp92gdtwJc4dOMI8h7jmYWdtKGz5atekVTi26QW08hRvU2j3BXC33HMysLZROTS0ePjq9YxNr+mawIi8ev/Ysprw+yA1qBjg4mFmbKL15LWv4qLD+UWvfoDYaPKxkZi0vbXVUDx/Vxj0HM2t5aWsfDbm+kQ3KwcHMWkpabiHzaWktvr5RI9XrYT/nS9osqVvSkpT9t0pal7xekPR60b69RftW1qM9Zta+Su9b8M1rI6Pm4CCpA/gKcAFwPHCJpOOL60TEpyLipIg4CfgycF/R7j/274uIebW2x8zaV1ZuwTev1V89hpVmA90R8SKApLuB+cDGjPqXAJ+rw/ea2RiTmVvAN6/VWz2Cw1HAtqLt7cBpaRUlvQc4FnikqHiipDzQC9wcEf9ShzaZWQvLWk47M7cwaWJLL4/djOqRc1BKWWTUvRhYERF7i8qOiYgc8GHgS5Lem/ol0mJJeUn5np6e2lpsZk0tazntQXMLVlf1CA7bga6i7aOBHRl1Lwa+V1wQETuS9xeBx4CT0w6MiGURkYuI3OTJk2tts5k1qbS8AuDcwiirx7DSs8B0SccCv6UQAD5cWknSccAhwFNFZYcAb0bEHkmHAWcAX6hDm8ysRaXlFZYumOncwiirOThERK+kK4GHgA7gjojYIOkGIB8R/dNTLwHujojiIacZwNcl9VHoxdwcEVmJbDNrM1Xds5DkHpxbGB11uQkuIh4EHiwp+4eS7X9MOe5JYFY92mBmrSdtPaS05zX377fR47WVzKwhfM9Cc/PyGWbWEGnPWnBeoXm452BmI670Oc5Zj+oceM6zl7xoOAcHMxtxaeshnchm7ppwE1d33stdE25iFpt9z0ITcXAwsxE1rGctWMM552BmIyozt+BnLTQ19xzMbMT09xre3lsIDm/vjX25hf5nLZx9feHdgaGpODiYWd2UJp77ew2n6AWu6HiAU/TCQO8BcOK5iTk4mFndlCaeV216hVl9JYnnvs3OLbQABwczq4u0xPPT183lvgv7OHDcXjpVeL/vwr7kfgZrZg4OZlYXaYlnoPDs5o4JoI7C+9QzG9hKq5SDg5nVzInn9uPgYGZVc+K5/Tk4mFnVnHhufw4OZlYVJ57HhroEB0nnS9osqVvSkpT9l0nqkbQueS0q2rdQ0pbktbAe7TGzkePE89hQ8/IZkjqArwDnUHie9LOSVqY80e2eiLiy5NhDgc8BOSCAtcmxr9XaLjOrv6zE81VzpjGlP/G89YlCYHB+oaXVo+cwG+iOiBcj4i3gbmB+hceeB6yKiFeTgLAKOL8ObTKzEeDE89hRj+BwFLCtaHt7Ulbqg5Kek7RCUleVx5rZKCudkQROPI8l9QgOSimLku0fAFMj4kTgYWB5FccWKkqLJeUl5Xt6eobdWDOrTOmMJMCJ5zGkHsFhO9BVtH00sKO4QkT8PiL2JJvfAP680mOL/sayiMhFRG7y5Ml1aLaZZUmbkTTAiecxoR7B4VlguqRjJU0ALgZWFleQdGTR5jxgU/L5IeBcSYdIOgQ4NykzswbKnJEEvuN5jKh5tlJE9Eq6ksKPegdwR0RskHQDkI+IlcBVkuYBvcCrwGXJsa9K+mcKAQbghoh4tdY2mdnwDTojadLEQqWu2Q4KbU4RqUP8TS2Xy0U+n290M8zaws5du7noa0+x4uPvZ8qkiXz2/vXck9/GrL7NnD5uE2v6ZrB+3HF86NRjWLpgZqObazWQtDYicpXU9R3SZmOcl8KwNA4OZmOYl8KwLA4OZmOYl8KwLA4OZmOUn8Fgg3FwMBsjsp7BUMxLYVg/BwezMSIt8fz23v3XSXp7bzjxbEAd7nMws+ZXmni+as60QoJ52zOw/HLY+1Yht+AhJEu452A2BmQmnrc+UQgMsbfwvvWJBrbSmomDg1mbGzTx7FlJlsHDSmZtbrDE89IFfkCPpXPPwazNlM5KGjLx7FlJlsI9B7M2UzwraemCmU4827C452DWRjKfw+DEs1XJwcGsjXg5DKsXBwezNuHlMKye6hIcJJ0vabOkbklLUvZfLWmjpOckrZb0nqJ9eyWtS14rS481s8p4OQyrp5qDg6QO4CvABcDxwCWSji+p9nMgFxEnAiuALxTt+2NEnJS85tXaHrOxImtWUjEvh2HDVY/ZSrOB7oh4EUDS3cB8YGN/hYh4tKj+GuAjdfheszEtdVYSFGYm+b4Fq1E9hpWOArYVbW9PyrJcDvy4aHuipLykNZIW1KE9Zm0vc1bStmdg+Tx45MbC+7ZnGttQa1n1CA5KKUt9MLWkjwA54ItFxcckzzT9MPAlSe/NOHZxEkTyPT09tbbZrKV5rSQbafUIDtuBrqLto4EdpZUkzQWuB+ZFxJ7+8ojYkby/CDwGnJz2JRGxLCJyEZGbPHlyHZpt1pq8VpKNhnoEh2eB6ZKOlTQBuBjYb9aRpJOBr1MIDDuLyg+RdEDy+TDgDIpyFWZWbtBZSZ6yanVSc3CIiF7gSuAhYBPw/YjYIOkGSf2zj74I/Alwb8mU1RlAXtIvgEeBmyPCwcGsSNWzkjxl1epAEanpgaaWy+Uin883uhlmo+Kz96/nrmd+w9+d9h6WLpi5b4dnJVmVJK1NcrxD8h3SZk3Ms5KsURwczJqYZyVZozg4mDUpz0qyRvLzHMyalJ/gZo3knoNZk/CsJGsm7jmYNQmvlWTNxD0HsybgWUnWbBwczJqAZyVZs3FwMGswz0qyZuScg1mDeVaSNSP3HMxGUemMJNg3K+kUvcAVHQ9wil7wrCRrOPcczEZR6YwkoDAradszsPzyQl6hY4JXVLWGc8/BbJRkzkgCJ56t6Tg4mI2SzBlJ4MSzNR0HB7NRMOiMJPBDeqzp1CU4SDpf0mZJ3ZKWpOw/QNI9yf6nJU0t2veZpHyzpPPq0R6zZjPo09v6OfFsTaTm4CCpA/gKcAFwPHCJpONLql0OvBYR04BbgVuSY4+n8FjRE4Dzgf+d/D2zlpa1TlLmjCSzJlOP2Uqzge6IeBFA0t3AfPZ/FvR84B+TzyuA/yVJSfndEbEH+LWk7uTvPVWHdpk1TOo6SZ6RZC2kHsNKRwHbira3J2WpdZJnTv8BeFeFx5q1lMxZSZ6RZC2kHsFBKWWlD6bOqlPJsYU/IC2WlJeU7+npqbKJZqMnc1aSZyRZC6lHcNgOdBVtHw3syKojqRP4U+DVCo8FICKWRUQuInKTJ0+uQ7PN6m/QWUmekWQtpB7B4VlguqRjJU2gkGBeWVJnJbAw+XwR8EhERFJ+cTKb6VhgOuA1ia1lDTkryTOSrEXUHBySHMKVwEPAJuD7EbFB0g2S5iXVvgm8K0k4Xw0sSY7dAHyfQvL6J8AnImJvrW0yGy2elWTtShGpQ/xNLZfLRT6fb3QzzPjs/eu565nf8HenvWdgraSBB/R4VpI1GUlrIyJXSV3fIW02TJ6VZO3MwcFsmDwrydqZg4PZMHhWkrU7BwezYfCsJGt3Dg5mFfCsJBtr/CQ4swp4rSQba9xzMBuCZyXZWOTgYDYEz0qyscjBwWwQnpVkY5WDg9kgPCvJxioHB7MinpVkVuDZSmZFPCvJrMA9B7OEZyWZ7ePgYJbwrCSzfRwczPCsJLNSDg5meFaSWamagoOkQyWtkrQleT8kpc5Jkp6StEHSc5I+VLTv25J+LWld8jqplvaYVcqzkswGV+tspSXA6oi4WdKSZPvakjpvApdGxBZJ7wbWSnooIl5P9n86IlbU2A6zqnhWktngah1Wmg8sTz4vBxaUVoiIFyJiS/J5B7ATmFzj95oNm2clmQ2t1uBweES8BJC8TxmssqTZwATgV0XFNybDTbdKOmCQYxdLykvK9/T01NhsG8s8K8lsaIqSJFxZBelh4IiUXdcDyyPi4KK6r0VEWd4h2Xck8BiwMCLWFJW9TCFgLAN+FRE3DNXoXC4X+Xx+qGpmZXbu2s2ZX3iUPb19A2UTO8fx+LVnMWXSxMLQ0tYnCoHBQ0rWZiStjYhcJXWHzDlExNxBvugVSUdGxEvJD/3OjHoHAT8CPtsfGJK//VLycY+kbwF/X0mjzYarv9dwil7g9HGbWNM3g/Vx3EDuga7ZDgpm1D6stBJYmHxeCDxQWkHSBOB+4M6IuLdk35HJuyjkK56vsT1mA0pnJEFhVtKsvs3cNeEmru68l7sm3MSsvs2elWRWotbgcDNwjqQtwDnJNpJykm5P6vwt8AHgspQpq3dJWg+sBw4DltbYHrMBxTOS+j193Vzuu7CPA8ftpVOF9/su7CvMVjKzATVNZY2I3wNzUsrzwKLk83eA72Qcf3Yt32+WpXRG0lVzphVyCrAv8dw/ZdWJZ7MyvkPa2lLmjCTwchhmFXBwsLYz6DpJ/bwchtmgHBys7RTPSOpfCqOs92Bmg3JwsJaXtk6SZySZ1cbBwVpe6awkz0gyq52Dg7W0zHWSvBSGWU0cHKylZc5K8owks5o4OFjLGnJWkmckmQ2bg4O1jNLEs2clmY0cBwdrGaWJZ89KMhs5Dg7WEtISz56VZDZyHBysJfgBPWajy8HBmt6giWfPSjIbEQ4O1vSGTDx7VpJZ3Tk4WNPxchhmjVdTcJB0qKRVkrYk71nPj95b9KCflUXlx0p6Ojn+nuSpcTbGeTkMs8arteewBFgdEdOB1cl2mj9GxEnJa15R+S3ArcnxrwGX19gea3FeDsOsOdQaHOYDy5PPyyk8B7oiyXOjzwZWDOd4a09eDsOsOdQaHA6PiJcAkvcpGfUmSspLWiOpPwC8C3g9InqT7e3AUVlfJGlx8jfyPT09NTbbmlF/r2FW32au6HiAWX2bvRyGWYMMGRwkPSzp+ZTX/Cq+55iIyAEfBr4k6b2AUupF1h+IiGURkYuI3OTJk6v4amtWacthnEhJ4pnNXg7DrAGGDA4RMTciZqa8HgBekXQkQPK+M+Nv7EjeXwQeA04GfgccLKkzqXY0sKPmM7KWkbYcxqmxkfH00qk+xtPLqbHRs5LMGqDWYaWVwMLk80LggdIKkg6RdEDy+TDgDGBjRATwKHDRYMdbe8paDuOajy2ic/wBoA46xx/ANR9b5FlJZg1Qa3C4GThH0hbgnGQbSTlJtyd1ZgB5Sb+gEAxujoiNyb5rgasldVPIQXyzxvZYi3Di2ay5KSJzmL9p5XK5yOfzjW6GDdPOXbs58wuPcsLeX3L6uE2s6ZvBxo738fi1ZzFl0sRGN8+sbUlam+R/h+Q7pG3EOfFs1nocHGzEOfFs1no8rGQjqn8IaU9vHxM7x+0bOtr2DCyfB3vfKtzx7PyC2YjzsJI1DSeezVpT59BVzIan+I7n0zsKiecVeXHVnGmF3kPXbAcFsyblnoPVjRPPZu3DwcHqxolns/bhhLTVhRPPZs3PCWkbdU48m7UXBwerSmleob/MS22btRcHB6tKaV6hv8yJZ7P24uBgFct6hKcTz2btx8HBKpaVV/BS22btx8HBUpXmFirKKzjxbNY2HBwsVWluoaK8ghPPZm2jpuAg6VBJqyRtSd4PSalzlqR1Ra/dkhYk+74t6ddF+06qpT1WH2m5BecVzMaWWtdWWgKsjoibJS1Jtq8trhARjwInQSGYAN3AT4uqfDoiVtTYDqujtNzC09fNhW0HwfKVsPctOjsncM2iRVzjXoJZW6p1WGk+sDz5vBxYMET9i4AfR8SbNX6v1UlVuQXnFczGjFqDw+ER8RJA8j5liPoXA98rKbtR0nOSbpV0QI3tsSpVnVtwXsFsTBgyOEh6WNLzKa/51XyRpCOBWcBDRcWfAd4HnAocSsmQVMnxiyXlJeV7enqq+WrL4NyCmWUZMucQEZmT1SW9IunIiHgp+fHfOcif+lvg/oh4u+hvv5R83CPpW8DfD9KOZcAyKCy8N1S7bWjOLZhZllqHlVYCC5PPC4EHBql7CSVDSklAQZIo5Cuer7E9lsG5BTOrRq3B4WbgHElbgHOSbSTlJN3eX0nSVKAL+L8lx98laT2wHjgMWFpjeyyDcwtmVo2agkNE/D4i5kTE9OT91aQ8HxGLiuptjYijIqKv5PizI2JWRMyMiI9ExBu1tMfSObdgZtXyM6THAOcWzKxaXj6jzTi3YGb14ODQZpxbMLN6cHBoI84tmFm9ODi0qLTHdWblFvysBTOrloNDiyodPnJuwczqycGhyaX1ENKGj5xbMLN6cnBocqU9hP6y0uEj5xbMrJ58n0MTK+0hXDVnGgQDw0end2xiTd8MVuTF49eexZTXfd+CmdWHg0MT2LlrNxd97SlWfPz9TJk0caA8rYdABCeymTsn3MR4enmbTi7dez23re5i6YIkt7D1CZh6poeQzGzYPKzUBNKGjrISzA9tHGL4yLkFM6sDB4dRVGlyGbJvXjvvhCM8NdXMRpyDwyjKSi6fSNI7KJpdNGiC2VNTzWyEOecwAtJyCFnJ5S1rH+HOjhsH8gcfzYudc6YNvTBe12wHBTMbMe451CBtmAgq7yHctnoLp2rDfr2DnDbsf2+Cewhm1gA1BQdJfyNpg6Q+SblB6p0vabOkbklLisqPlfS0pC2S7pE0oZb21Crrx76aIJCWQ9i5a/dAD+Hqznu5s+NGuvOreWjjKzzZO4O36aQ3xvE2nTzZO2P/exOcYDazBqi15/A88J+Bx7MqSOoAvgJcABwPXCLp+GT3LcCtETEdeA24vMb2VKSaH/us8qESyZX0EM474Qju+/ynOHDRj+ic+1kOXPQj7vv8p5xcNrOGq/VJcJsiYvMQ1WYD3RHxYkS8BdwNzE+eG302sCKpt5zCc6RHXDU/9tUEgWH3ENw7MLMmMxoJ6aOAbUXb24HTgHcBr0dEb1H5USPdmP4f8I+P28DP8iewc840pkyaOPBjf2rHRp7l+OSmspmp5VedPS01kXzLW7379RCIQg9h2glzWLpgLmw7A7Y+QefUM7nPgcDMmtiQwUHSw8ARKbuuj4gHKvgOpZTFIOVZ7VgMLAY45phjKvjadPevvJ9vdyxNftTv586Vh/PX/2lB6o/9xtldVQWBb64/jhm9M/jEhE6I3oEewm83vszSBTM9w8jMWsaQwSEiah0A3w50FW0fDewAfgccLKkz6T30l2e1YxmwDCCXy2UGkcHs3LWbNzY/xvhx+37U//2Xj3LL+Pel/th/8u51nFdFEDhoUif3Xfcp9xDMrOWNxrDSs8B0SccCvwUuBj4cESHpUeAiCnmIhUAlPZFhu231FjbHDK5g34/6UzGD59e/nPpj/6ueN3iSYQQB9xDMrMUpYlj/CC8cLP018GVgMvA6sC4izpP0buD2iLgwqXch8CWgA7gjIm5Myv+MQmA4FPg58JGI2DPU9+Zyucjn81W397SbHuaVXXs4RS9w+rjCiqY/i//A4QcdkNx09kz6onVZ5WZmLUTS2ojIvO1gv7q1BIdGGW5wMDMby6oJDr5D2szMyjg4mJlZGQcHMzMr4+BgZmZlHBzMzKyMg4OZmZVxcDAzszIODmZmVqYlb4KT1AP8a6PbkeIwCmtGtbJWP4dWbz/4HJpFO57DeyJiciUHtmRwaFaS8pXefdisWv0cWr394HNoFmP9HDysZGZmZRwczMysjINDfS1rdAPqoNXPodXbDz6HZjGmz8E5BzMzK+Oeg5mZlXFwqIKkiZKekfQLSRsk/VNKnQMk3SOpW9LTkqaOfkuzVXgOl0nqkbQueS1qRFuHIqlD0s8l/TBlX1Nfh35DnEPTXwdJWyWtT9pX9pAVFdyWXIfnJJ3SiHYOpoJz+AtJfyi6Dv/QiHYORtLBklZI+qWkTZLeX7K/6uswGo8JbSd7gLMj4g1J44H/J+nHEbGmqM7lwGsRMU3SxcAtwIca0dgMlZwDwD0RcWUD2leNTwKbgINS9jX7deg32DlAa1yHsyIi636AC4Dpyes04KvJe7MZ7BwAnoiIvxq11lTvfwI/iYiLJE0A3lGyv+rr4J5DFaLgjWRzfPIqTdrMB5Ynn1cAcyRplJo4pArPoelJOhr4S+D2jCpNfR2gonNoB/OBO5P/79YAB0s6stGNaieSDgI+AHwTICLeiojXS6pVfR0cHKqUDAOsA3YCqyLi6ZIqRwHbACKiF/gD8K7RbeXgKjgHgA8m3c8VkrpGuYmV+BJwDdCXsb/prwNDnwM0/3UI4KeS1kpanLJ/4DoktidlzWSocwB4fzIU+2NJJ4xm4yrwZ0AP8K1kiPJ2Se8sqVP1dXBwqFJE7I2Ik4CjgdmSZpZUSfvXaVP9y7yCc/gBMDUiTgQeZt+/wJuCpL8CdkbE2sGqpZQ1zXWo8Bya+jokzoiIUygMW3xC0gdK9jf1dUgMdQ4/o7DsxH8Evgz8y2g3cAidwCnAVyPiZODfgSUldaq+Dg4Ow5R02x4Dzi/ZtR3oApDUCfwp8OqoNq5CWecQEb+PiD3J5jeAPx/lpg3lDGCepK3A3cDZkr5TUqfZr8OQ59AC14GI2JG87wTuB2aXVBm4DomjgR2j07rKDHUOEbGrfyg2Ih4Exks6bNQbmm07sL1oBGAFhWBRWqeq6+DgUAVJkyUdnHw+EJgL/LKk2kpgYfL5IuCRaKKbSSo5h5KxyHkUEqZNIyI+ExFHR8RU4GIK/40/UlKtqa9DJefQ7NdB0jslTer/DJwLPF9SbSVwaTJb5nTgDxHx0ig3NVMl5yDpiP58laTZFH43fz/abc0SES8D2yQdlxTNATaWVKv6Oni2UnWOBJZL6qDwP8j3I+KHkm4A8hGxkkJS6P9I6qbwL9WLG9fcVJWcw1WS5gG9FM7hsoa1tgotdh1Stdh1OBy4P/nd7AS+GxE/kfQxgIj4GvAgcCHQDbwJfLRBbc1SyTlcBHxcUi/wR+DiZvqHRuK/A3clM5VeBD5a63XwHdJmZlbGw0pmZlbGwcHMzMo4OJiZWRkHBzMzK+PgYGZWI0n/nNzJvk7STyW9O6PeTyS9rpSFFpP9X5b0RtH2ByT9TFKvpIuqaE/mgo6VcnAwM6uCCqu0fruk+IsRcWKy8sAPgayVW78I/JeMv5sDDi4p/g2FKczfrbKZ/Qs6DpuDg5lZjSJiV9HmO8lYmiIiVgP/Vlqe3Hf0RQprbRXX3xoRz5Gy/pakT0t6Numx/FNReV0WdPRNcGZmdSDpRuBSCos8nlXl4VcCKyPipUoWD5Z0LoXlt2dTWDdppaQPRMTj7FvQcVKVbdiPew5mZhVQ4aFR6yj8i3ye9j385zyAiLg+IrqAuyj82Ff6d98N/A2FRf0qdW7y+jmFhQHfB0yvcEHHirjnYGZWgYg4DQo5B+CyiLgso+p3gR8Bn6vwT58MTAO6k17DOyR1R8S0QY4R8PmI+Pp+hdLnKQSuC4GJwEGSvpOy9tiQ3HMwM6uRpOlFm/MoX5AzU0T8KCKOiIipyUKMbw4RGAAeAv6rpD9Jvv/Be1qLAAAAhUlEQVQoSVMqXJSyIg4OZma1u1nS85KeozDc80kozECSNJAYlvQEcC+FJxNu7x+SyiLpVEnbKQw7fV3SBoCI+CmFHspTktZTWKa7phxD2Xd74T0zMyvlnoOZmZVxcDAzszIODmZmVsbBwczMyjg4mJlZGQcHMzMr4+BgZmZlHBzMzKzM/wfczdF4b7QbjAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "lstmDemo = LstmDemo(input_dim,hidden_size,num_layers).cuda()\n",
    "optimizer = torch.optim.Adam(lstmDemo.parameters(), lr=lr)\n",
    "loss_func = nn.MSELoss()\n",
    "\n",
    "h_c_state = (torch.zeros(num_layers,1,hidden_size).cuda(),torch.zeros(num_layers,1,hidden_size).cuda())\n",
    "for step in range(epochs):\n",
    "    start, end = step * np.pi, (step + 1) * np.pi  # 时间跨度\n",
    "    # 使用Sin函数预测Cos函数\n",
    "    steps = np.linspace(start, end, bins, dtype=np.float32, endpoint=False)\n",
    "    x_np = np.sin(steps)\n",
    "    y_np = np.cos(steps)\n",
    "    x = torch.from_numpy(x_np).unsqueeze(1).unsqueeze(2).cuda()#【100，1,1】尺寸大小为(time_step, batch, input_size)\n",
    "    y = torch.from_numpy(y_np).unsqueeze(1).unsqueeze(2).cuda()#【100，1,1】\n",
    "    prediction, h_state = lstmDemo(x,h_c_state)  # RNN输出（预测结果，隐藏状态）\n",
    "    #将每一次输出的中间状态传递下去(不带梯度)\n",
    "    h_c_state = (h_state[0].detach(),h_state[1].detach())\n",
    "    loss = loss_func(prediction, y)\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    if(step%100==0):\n",
    "        print(\"loss:{:.8f}\".format(loss))\n",
    "plt.scatter(steps,y_np,marker=\"^\")\n",
    "plt.scatter(steps, prediction.cpu().data.numpy().flatten(),marker=\".\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GRU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class GRUDemo(nn.Module):\n",
    "    def __init__(self,input_dim,hidden_size,num_layers):\n",
    "        super().__init__()\n",
    "        self.input_dim = input_dim\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_layers = num_layers\n",
    "        self.gru = nn.GRU(\n",
    "            input_size=input_dim,\n",
    "            hidden_size=hidden_size,\n",
    "            num_layers=num_layers\n",
    "        )\n",
    "        self.out = nn.Linear(hidden_size, 1)\n",
    "\n",
    "    def forward(self, x, h):\n",
    "        r_out, h_state = self.gru(x,h)\n",
    "        outs = [] \n",
    "        for record in range(r_out.size(1)):\n",
    "            outs.append(self.out(r_out[:, record, :]))\n",
    "        return torch.stack(outs, dim=1), h_state\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss:0.50176054\n",
      "loss:0.00226786\n",
      "loss:0.00002110\n",
      "loss:0.00734446\n",
      "loss:0.00001271\n",
      "loss:0.00001133\n",
      "loss:0.00525041\n",
      "loss:0.00001578\n",
      "loss:0.00000526\n",
      "loss:0.00047405\n",
      "loss:0.00038403\n",
      "loss:0.00005233\n",
      "loss:0.01299966\n",
      "loss:0.00000396\n",
      "loss:0.00000987\n",
      "loss:0.00000983\n",
      "loss:0.00179682\n",
      "loss:0.00026669\n",
      "loss:0.00002136\n",
      "loss:0.00000122\n",
      "loss:0.00000076\n",
      "loss:0.00000065\n",
      "loss:0.00273025\n",
      "loss:0.00008161\n",
      "loss:0.00000101\n",
      "loss:0.00024650\n",
      "loss:0.00003742\n",
      "loss:0.00002226\n",
      "loss:0.00000259\n",
      "loss:0.00000088\n",
      "loss:0.00034960\n",
      "loss:0.00017886\n",
      "loss:0.00002973\n",
      "loss:0.00018834\n",
      "loss:0.00029263\n",
      "loss:0.00006494\n",
      "loss:0.00016715\n",
      "loss:0.00178642\n",
      "loss:0.50297040\n",
      "loss:0.12733065\n",
      "loss:0.00891744\n",
      "loss:0.00289894\n",
      "loss:0.00046954\n",
      "loss:0.00041219\n",
      "loss:0.00169624\n",
      "loss:0.00013055\n",
      "loss:0.00030379\n",
      "loss:0.00054660\n",
      "loss:0.00002946\n",
      "loss:0.00008561\n",
      "loss:0.00295883\n",
      "loss:0.00280066\n",
      "loss:0.00009799\n",
      "loss:0.00000955\n",
      "loss:0.00001924\n",
      "loss:0.00011282\n",
      "loss:0.03090166\n",
      "loss:0.00353032\n",
      "loss:0.00003653\n",
      "loss:0.00002000\n",
      "loss:0.00001959\n",
      "loss:0.00001776\n",
      "loss:0.00001586\n",
      "loss:0.00103418\n",
      "loss:0.05394575\n",
      "loss:0.00105088\n",
      "loss:0.00007497\n",
      "loss:0.00005789\n",
      "loss:0.00005236\n",
      "loss:0.00005393\n",
      "loss:0.00624973\n",
      "loss:0.00650680\n",
      "loss:0.00447993\n",
      "loss:0.00424718\n",
      "loss:0.00007916\n",
      "loss:0.00010222\n",
      "loss:0.00243222\n",
      "loss:0.00290016\n",
      "loss:0.00011132\n",
      "loss:0.00325956\n",
      "loss:0.00022088\n",
      "loss:0.00068554\n",
      "loss:0.00071156\n",
      "loss:0.00304861\n",
      "loss:0.00017071\n",
      "loss:0.00124577\n",
      "loss:0.00116748\n",
      "loss:0.00067161\n",
      "loss:0.00332054\n",
      "loss:0.00629372\n",
      "loss:0.00845968\n",
      "loss:0.00158721\n",
      "loss:0.00098066\n",
      "loss:0.00080520\n",
      "loss:0.00704986\n",
      "loss:0.00153961\n",
      "loss:0.00021508\n",
      "loss:0.00093294\n",
      "loss:0.00388560\n",
      "loss:0.00572870\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEJCAYAAAB/pOvWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3X+UlNWd5/H3p2lQE6NgBDUI4qytQbNZk2lJcrJmE02iyRgxo25IzIrRHJKMrjmb+aHBOZsdgzvGnMEZZvNDJxrJDBGVCSOYH4YgbHrPKFAQRoSmA3EYuwelybSKjEJ3dX/3j3q6Kaqruqup6q4f/XmdU6eq7nOfrvtYnvpy7/c+9yoiMDMzy9ZQ6QaYmVn1cXAwM7NBHBzMzGwQBwczMxvEwcHMzAZxcDAzs0EcHMzMbBAHBzMzG8TBwczMBmmsdAOOxamnnhqzZs2qdDPMzGrK5s2bfxsRU4upW5PBYdasWaRSqUo3w8yspkj6l2LreljJzMwGcXAwM7NBHBzMzGwQBwczMxvEwcHMzAZxcDAzs0HKEhwkPSipU9JzBY5L0hJJuyU9K+ndWcfmS9qVPOaXoz1mZlaacvUcHgIuH+L4x4Cm5LEA+A6ApFOArwHvAeYAX5M0pUxtMjOrKZ0HDvGBe9bR+dqhospHU1mCQ0T8Eugaospc4AeR8QwwWdIZwGXAmojoioiXgTUMHWTMzGpeoR/7JWt30f7y6yxZu7uo8tE0VjmH6UB71vuOpKxQ+SCSFkhKSUrt379/1BpqZlYuIwkCnQcOsWvzU3yp4XF2p9YOnFOofLSNVXBQnrIYonxwYcT9EdEcEc1Tpxa1NIiZWUWNJAisXLWShyYs4iuNj/H9CYtYueofhiwfbWMVHDqAGVnvzwT2DlFuZlYThsoTFBsEOg8c4mDbeiaSplF9TCTNv+9cx469r+YtH4vew1gFh1XA9cmspfcCr0bEi8CTwEclTUkS0R9NyszMakKhfMBIgsA3fraTZ2I2PTSSjgZ6aOTpmM2Xl2/NWz4WuYeyrMoq6WHgg8CpkjrIzECaCBAR3wV+Anwc2A28DnwuOdYl6evApuRP3RkRQyW2zcwqovPAIa757tOs+NL7mPaW4wfKMr2D7WxJXUDnpecw7S3HHwkCDZkgQCRBYOLbeSFm8wc0QqQHfuyf2/YSb6SbuK53Ie9taOWZvtlsiSYa9h+kLwaXv7DjJRZd9Y5Rvd6yBIeI+PQwxwO4ucCxB4EHy9EOM7PRkt1D6P9h7u8dTCRNDyv5warT+MJ181iydhdtIwgCp53USOvXLwd+r8CnFyofPTW5n4OZ2VjK10MgyNs76HztKta07mPfMQWB6uHgYGaWJd/wUb4eQvubLsjbO+hau5sNCz+c/LXqDwKFeG0lM7MsK1et5IoDDw9MGS2URH5yxz42pZu4rnshi9PXcl33Qjalm1iz46UKX0F5uOdgZuNSvh5C184Wrt91KxMnpOn59Uq62qazZMfkvD2Eyy44Pck91G7vYCjuOZjZuJTbQwBIrV99VA8htX41a1rru4dQiHsOZjbu5OshpM+4iAc6pnPxhCM9hAfbp7P6tv+c9Czqs4dQiIODmY07qfWruYQjs4xa1q/ml9MmsyUyPYT+GUbbGpqOmro6nnhYyczqWtfOFu6762a62lqATK7hgY7pR911/GD7dJ7csY+e3mBLnMu3e+eyJc6lpzfqfvioEPcczKx+tW/kxEeu5qa+HvqWPwY3PsGS1Al5ewifGkgwG7jnYGZ17GDbOhr6emhUHw19PRxsW8eaVvcQiuGeg5nVha6dLTz298u59pp5nHLexQAse2km12dNQV2276ysG9RsKA4OZlb78gwfdZ78ThbvnMyTWUtY7Gg9mU++dmjgvgYrzMHBzGrewbZ1HJ8MH6WT4aMlr51AX2SGj7b0ngvAxIhxO/topJxzMLOat+ylmUfNPlq276yB3EI25xaK556DmdWU3NxC54FDeYePfjlw85odi7L0HCRdLqlN0m5Jt+c5fq+krcnj15JeyTrWm3VsVTnaY2Z1qj+30P1DTlx+NbRvZMnaXQPDR/2zj3qT4SM7diUHB0kTgG8BHwPOBz4t6fzsOhHxPyLiwoi4EPhr4EdZh9/oPxYRV5baHjOrX0NNTc3m4aPSlWNYaQ6wOyKeB5C0HJgL7ChQ/9NkthE1MxsRT00dO+UYVpoOtGe970jKBpF0FnA28FRW8fGSUpKekXRVGdpjZjUud8kLYCC3kL066r2tJ9P52qEKtrR+lSM4KE9Z5CkDmAesiIjerLKZEdEMfAb4S0n/Ie+HSAuSIJLav39/aS02s+qVJ68AOLcwxsoRHDqAGVnvzwT2Fqg7D3g4uyAi9ibPzwPrgXflOzEi7o+I5ohonjp1aqltNrMqlS+vADi3MMbKkXPYBDRJOhv4VzIB4DO5lSSdB0wBns4qmwK8HhGHJZ0KvB+4pwxtMrMalS+v8AVwbmGMlRwcIiIt6RbgSWAC8GBEbJd0J5CKiP7pqZ8GlkdEduifDdwnqY9ML+buiCiUyDazOlPsPQte8mLs6ejf6trQ3NwcqVSq0s0ws1K0b6T7wSto6Ouhr2Eik258gj9NncAjqfajho8mThCfumiml7woA0mbkxzvsLx8hplVhO9ZqG5ePsPMKsL3LFQ3BwczG33tG2FPC8y6GGbMcW6hBjg4mNnoat9ILP0EveluJjROQvNXsyTl5bSrnXMOZja69rTQl+6mkT760t2wp8W5hRrgnoOZjaquqXM4IRqZSCa38Ma0OWxYeHGlm2XDcHAws1G1uHUybb13cFHsYJPO57zWKSw6r9KtsuE4OJhZ+eRJPD+2uYPD6SY20QTAtlQ7t156jhPPVc7BwczKY4jEc7ZeJ55rghPSZlYeTjzXFfcczKwsnHiuLw4OZlYWTjzXFwcHMxs5J57rnoODmY2ME8/jghPSZjYyTjyPC+45mNmIOPE8PpSl5yDpckltknZLuj3P8Rsk7Ze0NXl8PuvYfEm7ksf8crTHzEbP4tbJXN97B4vT12aeW6dUukk2CkruOUiaAHwL+AjQAWyStCrPdp+PRMQtOeeeAnwNaAYC2Jyc+3Kp7TKz8nPiefwoR89hDrA7Ip6PiG5gOTC3yHMvA9ZERFcSENYAl5ehTWZWoq6dLdx31810tbUMlC1Zu6tg4tnqSzmCw3SgPet9R1KW62pJz0paIWnGCM9F0gJJKUmp/fv3l6HZZlZQ+0ZOfORqbur+IScuvzozdRWceB5HypGQVp6yyHm/Gng4Ig5L+iKwFLikyHMzhRH3A/cDNDc3561jZuVxsG0dxyf7O6eT/Z1PnDHH23iOI+XoOXQAM7Lenwnsza4QEf8WEYeTt38D/G6x55rZ2Fv20kx6aCQdDQP7O9v4Uo6ewyagSdLZwL8C84DPZFeQdEZEvJi8vRJoTV4/CfxvSf3THT4KfLUMbTKzY+T9nQ3K0HOIiDRwC5kf+lbg0YjYLulOSVcm1W6VtF3SPwG3Ajck53YBXycTYDYBdyZlZjZGchPP/UnnLXEu3+6dy5Y410nncUgRtTd839zcHKlUqtLNMKt97RvpfvAKGvp66GuYyKQbn+A9f3uAfQcOD6p62knHOedQ4yRtjojmYur6DmmzcSxf4nnDwtsq3SyrAl5byWwcc+LZCnHPwWyccuLZhuKeg9k45cSzDcXBwWycyJ2V5LudbSgeVjIbD/qXw+jroW/5Y3DjE555ZENyz8FsHDjYto6GZFZSQzIryWwoDg5m44BnJdlIeVjJrM55VpIdC/cczOqcZyXZsXBwMKsznpVk5eBhJbN64llJVibuOZjVEc9KsnJxcDCrI56VZOXiYSWzOuFZSVZOZek5SLpcUpuk3ZJuz3P8K5J2SHpW0lpJZ2Ud65W0NXmsKkd7zMaF9o3Q8heZZzwrycqr5J6DpAnAt4CPkNkTepOkVRGxI6var4DmiHhd0peAe4BPJcfeiIgLS22H2bjSvpFY+gl6091MaJyE5q9mTeuBgrOSFl31jgo11GpVOYaV5gC7I+J5AEnLgbnAQHCIiOys2DPAZ8vwuWbj154W+tLdNNKXCRB7Wtiw8A8r3SqrI+UYVpoOtGe970jKCrkJ+GnW++MlpSQ9I+mqMrTHrO51TZ1Dd2QSz93RSNe0OZVuktWZcvQclKcs78bUkj4LNAP/Jat4ZkTslfQ7wFOStkXEb/KcuwBYADBz5szSW21Wwxa3Tqat9w4uih1s0vmc1zqFRedVulVWT8rRc+gAZmS9PxPYm1tJ0oeBO4ArI2Jg9/KI2Js8Pw+sB96V70Mi4v6IaI6I5qlTp5ah2Wa1qfPAIR7b3MGmdBPf7p3LpnQTK1LtdL52qNJNszpSjuCwCWiSdLakScA84KhZR5LeBdxHJjB0ZpVPkXRc8vpU4P1k5SrMbLD+WUnZPCvJyq3k4BARaeAW4EmgFXg0IrZLulPSlUm1bwInAo/lTFmdDaQk/ROwDrg7Z5aT2bjntZKsEhSRNz1Q1ZqbmyOVSlW6GWajr30j3Q9eQUNfD30NE5l04xMww8lnOzaSNkdEczF1vXyGWRXzWklWKQ4OZlXMayVZpXhtJbMq5bWSrJLcczCrUl4rySrJwcGsSnhWklUTDyuZVQPv4GZVxj0HsyrgWUlWbRwczKqAZyVZtfGwklmFeVaSVSP3HMwqzLOSrBo5OJiNodwZSeBZSVadvLaS2VjxOklWYV5byawKeUaS1RIHB7Mx4hlJVks8W8lsDHhGktWasvQcJF0uqU3Sbkm35zl+nKRHkuMbJM3KOvbVpLxN0mXlaI9ZtfGMJKs1JQcHSROAbwEfA84HPi3p/JxqNwEvR8Q5wL3AN5JzzyezregFwOXAt5O/Z1bTvE6S1bpyDCvNAXZHxPMAkpYDczl6L+i5wP9KXq8A/o8kJeXLI+Iw8M+Sdid/7+kytMusMrxOktWBcgwrTQfas953JGV56yR7Tr8KvLXIc81qimclWT0oR3BQnrLcmycK1Snm3MwfkBZISklK7d+/f4RNNBs7npVk9aAcw0odwIys92cCewvU6ZDUCJwMdBV5LgARcT9wP2RugitDu83KzrOSrF6Uo+ewCWiSdLakSWQSzKty6qwC5ievrwGeisyt2auAeclsprOBJmBjGdpkVhGelWT1ouTgkOQQbgGeBFqBRyNiu6Q7JV2ZVHsAeGuScP4KcHty7nbgUTLJ658BN0dEb6ltMhsrnQcO8YF71tH52iHAs5KsfnhtJbMS3Pd3D/PqznWcPPsSvnDdvEo3x2xII1lbyXdImx2jrp0tXL/rViZOSNPz65V0tU3nlPMurnSzzMrCayuZHaPU+tVMJE2j+phImtT61ZVuklnZODiYHYPOA4d4oGP6UVNWH2yfPpB7MKt1Dg5mx2DJ2l1siSau617I4vS1XNe9kM3R5FlJVjccHMyKUGhWUvaUVc9KsnrihLRZEVauWskVB9axctU+vnDdPK+VZHXPwcFsGJ6VZOORh5XMhuFZSTYeOTiYDcGzkmy8cnAwG4JnJdl45eBglsWzkswynJA2y+JZSWYZDg5mCc9KMjvCw0pmCc9KMjvCwcEMz0oyy+XgYIZnJZnlKik4SDpF0hpJu5LnKXnqXCjpaUnbJT0r6VNZxx6S9M+StiaPC0tpj1mxPCvJbGgl7QQn6R6gKyLulnQ7MCUibsupcy4QEbFL0tuAzcDsiHhF0kPAExGxYiSf653grFTewc3Go7HcCW4u8MHk9VJgPXBUcIiIX2e93iupE5gKvFLiZ5sdE89KMhteqTmH0yLiRYDkedpQlSXNASYBv8kqvisZbrpX0nFDnLtAUkpSav/+/SU228Yzz0oyG96wwUHSLyQ9l+cxdyQfJOkM4G+Bz0VEX1L8VeDtwEXAKeT0OrJFxP0R0RwRzVOnTh3JR5sN8Kwks+IMO6wUEQVvEZW0T9IZEfFi8uPfWaDeScCPgT+NiGey/vaLycvDkr4P/NGIWm82Qtmzkt7b0MozfbPZ1pCZlbToqndUunlmVaPUYaVVwPzk9Xzg8dwKkiYBK4EfRMRjOcfOSJ4FXAU8V2J7zAbkzkgCz0oyK1aps5XeCjwKzAReAK6NiC5JzcAXI+Lzkj4LfB/YnnXqDRGxVdJTZJLTArYm5xwc7nM9W8mK4RlJZkcbyWylkoJDpTg42HC6drZwwsO/z0TS9NDIG5/5kWck2bg3kuDgO6StLnlGkllpHBys7nhGklnpHBys7nidJLPSOThYzeva2cJ9d91MV1sL4BlJZuXgzX6strVv5MRHruamvh76lj8GNz7h3dvMysA9B6tpB9vW0dDXQ6P6aOjr4WDbuko3yawuODhYTVv20syjEs/L9p1V6SaZ1QUPK1nN6jxwiMU7J/Nk75GlMHa0nswnXzvEtLccX+nmmdU09xysdrRvhJa/yDyTmZXUF0cnnnsjPCvJrAzcc7Da0L6RWPoJetPdTGichOavZk3rAXp6j77Dv39WkhfRMyuNg4PVhj0t9KW7aaQvEyD2tLBh4R9WulVmdcvDSlYTuqbOoTsyiefuaKRr2pxKN8msrrnnYDVhcetk2nrv4KLYwSadz3mtU1h0XqVbZVa/HBys+rRvhD0tMOtimDGHzgOHeGxzB4fTTWyiCYBtqXZuvfQcz0oyGyUODlZd8iSel6ROoC9nafn+WUlOPJuNjpJyDpJOkbRG0q7keUqBer2StiaPVVnlZ0vakJz/SLJrnI1nWYnnvnQ37GkZWCspm9dKMhtdpfYcbgfWRsTdkm5P3t+Wp94bEXFhnvJvAPdGxHJJ3wVuAr5TYpushnVNncMJ0Xhkk55pc9iw0Jv0mI21UmcrzQWWJq+XktkHuijJvtGXACuO5XyrT4tbJ3N97x0sTl+beW7N2xk1s1FWanA4LSJeBEiepxWod7yklKRnJPUHgLcCr0REOnnfAUwvsT1WS3LueO5PPG9KN/Ht3rlsSjexItXuTXrMKmDYYSVJvwBOz3PojhF8zsyI2Cvpd4CnJG0DDuSpV3BDa0kLgAUAM2fOHMFHW1Vy4tmsqg3bc4iID0fEO/I8Hgf2SToDIHnuLPA39ibPzwPrgXcBvwUmS+oPUGcCe4dox/0R0RwRzVOnTh3BJVpVcuLZrKqVmpBeBcwH7k6eH8+tkMxgej0iDks6FXg/cE9EhKR1wDXA8kLnW31y4tmsupUaHO4GHpV0E/ACcC2ApGbgixHxeWA2cJ+kPjI9lbsjYkdy/m3AckmLgF8BD5TYHqsRvuPZrLopouAwf9Vqbm6OVCpV6WZYsfLc8XzxPes4nO4bqHJ8YwO/vO1DvuPZbBRJ2hwRzcXU9R3SNrqceDarSV6V1UaXE89mNck9BxtVTjyb1SYHBxtVTjyb1SYHBysfL7VtVjccHKw8nHg2qytOSFt5OPFsVlfcc7CycOLZrL44OFhZOPFsVl8cHGxkcpLOgBPPZnXIOQcrXpJ0Tq9dRCz9xMA+DEvW7iqYeDaz2uTgYMXLk3QGnHg2q0MeVrKi5Us6nwJsWPjhSjfNzMrMwcGK5qSz2fjhYSXLq2tnC/fddTNdbZmhI+/vbDa+ODjYYO0bOfGRq7mp+4ecuPxqaN/opLPZOFNScJB0iqQ1knYlz1Py1PmQpK1Zj0OSrkqOPSTpn7OOXVhKe6w8Drato6Gvh0b10dDXw8G2dU46m40zpeYcbgfWRsTdkm5P3t+WXSEi1gEXQiaYALuBn2dV+eOIWFFiO6yMlr00k+tphMgknpftO8tJZ7NxptRhpbnA0uT1UuCqYepfA/w0Il4v8XOtTDoPHOID96wbyB10HjjE4p2Tua57IYvT13Jd90LubT3ZuQWzcabU4HBaRLwIkDxPG6b+PODhnLK7JD0r6V5JxxU6UdICSSlJqf3795fWahuwctVKrjjwMCtX/QNw5Ia2LXEu3+6dy5Y417kFs3FIkZNkHFRB+gVwep5DdwBLI2JyVt2XI2JQ3iE5dgbwLPC2iOjJKnsJmATcD/wmIu4crtHNzc2RSqWGq2bD6NrZwgkP//6R+xY+8yM+9veH2Xfg8KC6p510nIeWzGqcpM0R0VxM3WFzDhFR8BdB0j5JZ0TEi8kPfecQf+q/Aiv7A0Pyt19MXh6W9H3gj4pptJVHav1qLiFNo/og0rSsX82GhfdUullmVgVKHVZaBcxPXs8HHh+i7qfJGVJKAgqSRCZf8VyJ7bEC8uUWHuiYTg+NpKOBHhp5sH26cwtmBpQeHO4GPiJpF/CR5D2SmiV9r7+SpFnADOD/5py/TNI2YBtwKrCoxPZYAflyC1ui6ajE8+Zocm7BzIAicg7VyDmHkXFuwcygzDkHqy2dBw5xzXefZsWX3jewl4JzC2Y2Ul4+o84sWbuL9pdfHxgecm7BzI6Fg0Md6TxwiF2bn+JLDY+zO7WWztcOObdgZsfEw0p1ZOWqlTw0YVGSW1jJD1adxpp/OZWe3mAL57Kl99xMxWRNpEVXvaOyDTazquXgUKNycwudBw5xsG09ExuO5Bb+fec6Vv/JX3kfZzMbMQ8r1ajc3MKStbt4JmYflVt4OmZ7+MjMjomDQ5XLvXmtvyw3t7CmdR+b0kfnFjalm7yktpkdEw8rVbnsHkJ/jiBfbmHDwnnJGb9XucaaWd1wz6GK9W/NGcHAlpwDuYXkvoWJZHILnppqZuXk4FAF8g0dQabX8E7a+IMJj/MfaWPJ2t3OLZjZmPCwUhXIN3TUn1f4wYS7Bpa9+FxK/OaEC9ifbuK63oW8t6GVZ/pmsyWaeMFTU82sjBwcxlC+pS2OJJe3syV1AZ2XnsO0txzPkrW7uEjbB4aPiDTN2s45F1yaBAHnFsxs9HhYaRQMNUyUPf0UjiSXv9L4GN+fsGhg1dQ1rfv4x/TRw0f/mJ7t2UdmNibccxgFQw0TZfcQCPLeuNb52lXJyqgfhvb3w54WGmddzI9mzKnshZnZuOHgUIKRDBPlm37a/qYLaIvZ/AGNEOmB5HJXVlBhxpzMw8xsDJU0rCTpWknbJfVJKrhGuKTLJbVJ2i3p9qzysyVtkLRL0iOSJpXSnqK1b4SWv8g8Z+na2cJ9d91MV1tLUeW5G+j0l+UOExWafvrkDt+4ZmbVqdSew3PA7wP3FaogaQLwLTI7xXUAmyStiogdwDeAeyNiuaTvAjcB3ymxTUNr30gs/QS96W4mNE5C81dn/mXevpETH7mam/p66Fv+GNz4xJDlXTtbuH7XrUyckKbn1yvpaptO+oyL8g4TfWPi23khTw/hsgtOd3LZzKpSST2HiGiNiLZhqs0BdkfE8xHRDSwH5ib7Rl8CrEjqLSWzj/To2tNCX7qbRvroS3fDnkxv4GDbOhr6emhUHw19PRxsWzdkeWr96qN6Aqn1qwveg/CTbS+5h2BmNWUscg7Tgfas9x3Ae4C3Aq9ERDqrfPpoN6Zr6hxOiMYjW2ZOm8MpwLKXZnJ91r/sl+07iy8UKP9ksoHOxROOlD/YPp3f/HZf3nsQTjupkdavX457CGZWK4YNDpJ+AZye59AdEfF4EZ+hPGUxRHmhdiwAFgDMnDmziI/Nb3HrZNp67+Ci2MEmnc95rVO49YxDLN45mSezftR3tJ7MxXtfzVv+64k7BzbQ6S/f1tDEpzxMZGZ1YtjgEBGl7jbfAczIen8msBf4LTBZUmPSe+gvL9SO+4H7AZqbmwsGkaH0r1V0ON3EJpoA2JZq543uNH0RbIkjG+JMjODLy7fmLf/Jtpe8gY6Z1bWxGFbaBDRJOhv4V2Ae8JmICEnrgGvI5CHmA8X0RI7ZkrW76Iuj40pv1o99tp7e4Df7D9KXE4Z6eoNT3tw/TGRmVp9KCg6SPgn8NTAV+LGkrRFxmaS3Ad+LiI9HRFrSLcCTwATgwYjYnvyJ24DlkhYBvwIeKKU9w1nTui9vEPCPvZnZ0RRxTCM0FdXc3BypVKrSzTAzqymSNkdEwXvSsnltJTMzG8TBwczMBnFwMDOzQRwczMxsEAcHMzMbxMHBzMwGqcmprJL2A/9S6XbkcSqZO79rWa1fQ623H3wN1aIer+GsiJhazIk1GRyqlaRUsXOIq1WtX0Ottx98DdVivF+Dh5XMzGwQBwczMxvEwaG87q90A8qg1q+h1tsPvoZqMa6vwTkHMzMbxD0HMzMbxMFhBCQdL2mjpH+StF3Sn+Wpc5ykRyTtlrRB0qyxb2lhRV7DDZL2S9qaPD5fibYOR9IESb+S9ESeY1X9PfQb5hqq/nuQtEfStqR9g5ZKVsaS5Ht4VtK7K9HOoRRxDR+U9GrW9/A/K9HOoUiaLGmFpJ2SWiW9L+f4iL+Hsdjsp54cBi6JiIOSJgL/T9JPI+KZrDo3AS9HxDmS5gHfAD5VicYWUMw1ADwSEbdUoH0j8WWgFTgpz7Fq/x76DXUNUBvfw4ciotD9AB8DmpLHe4DvJM/VZqhrAGiJiCvGrDUj91fAzyLiGkmTgDflHB/x9+CewwhExsHk7cTkkZu0mQssTV6vAC6VlG+/7Ioo8hqqnqQzyWzW/b0CVar6e4CirqEezAV+kPx/9wyZrYHPqHSj6omkk4APkGyWFhHdEfFKTrURfw8ODiOUDANsBTqBNRGxIafKdKAdINkb+1XgrWPbyqEVcQ0AVyfdzxWSZuQ5Xml/CfwJ0FfgeNV/Dwx/DVD930MAP5e0WdKCPMcHvodER1JWTYa7BoD3JUOxP5V0wVg2rgi/A+wHvp8MUX5P0ptz6oz4e3BwGKGI6I2IC4EzgTmS3pFTJd+/TqvqX+ZFXMNqYFZEvBP4BUf+BV4VJF0BdEbE5qGq5Smrmu+hyGuo6u8h8f6IeDeZYYubJX0g53hVfw+J4a5hC5llJ/4TmW2R/2GsGziMRuDdwHci4l3AvwO359QZ8ffg4HCMkm7beiB38+kOYAaApEbgZKBrTBtXpELXEBH/FhGHk7d/A/zuGDdtOO8HrpS0B1gOXCLp73LqVPv3MOw11MD3QETsTZ47gZXAnJwqA99D4kxg79i0rjjDXUNEHOgfio2InwATJZ065g0trAPoyBoBWEEmWOTWGdH34OAwApKmSpqcvD4B+DCwM6faKmB+8voa4KmooptJirkYMGbRAAAC/ElEQVSGnLHIK8kkTKtGRHw1Is6MiFnAPDL/jT+bU62qv4dirqHavwdJb5b0lv7XwEeB53KqrQKuT2bLvBd4NSJeHOOmFlTMNUg6vT9fJWkOmd/NfxvrthYSES8B7ZLOS4ouBXbkVBvx9+DZSiNzBrBU0gQy/4M8GhFPSLoTSEXEKjJJob+VtJvMv1TnVa65eRVzDbdKuhJIk7mGGyrW2hGose8hrxr7Hk4DVia/m43ADyPiZ5K+CBAR3wV+Anwc2A28DnyuQm0tpJhruAb4kqQ08AYwr5r+oZH478CyZKbS88DnSv0efIe0mZkN4mElMzMbxMHBzMwGcXAwM7NBHBzMzGwQBwczsxJJ+npyJ/tWST+X9LYC9X4m6RXlWWgxOf7Xkg5mvf+ApC2S0pKuGUF7Ci7oWCwHBzOzEVBmldaHcoq/GRHvTFYeeAIotHLrN4H/VuDvNgOTc4pfIDOF+YcjbGb/go7HzMHBzKxEEXEg6+2bKbA0RUSsBV7LLU/uO/ommbW2suvviYhnybP+lqQ/lrQp6bH8WVZ5WRZ09E1wZmZlIOku4Hoyizx+aISn3wKsiogXi1k8WNJHySy/PYfMukmrJH0gIn7JkQUd3zLCNhzFPQczsyIos2nUVjL/Ir9SRzb/uQwgIu6IiBnAMjI/9sX+3bcB15JZ1K9YH00evyKzMODbgaYiF3QsinsOZmZFiIj3QCbnANwQETcUqPpD4MfA14r80+8CzgF2J72GN0naHRHnDHGOgD+PiPuOKpT+nEzg+jhwPHCSpL/Ls/bYsNxzMDMrkaSmrLdXMnhBzoIi4scRcXpEzEoWYnx9mMAA8CRwo6QTk8+fLmlakYtSFsXBwcysdHdLek7Ss2SGe74MmRlIkgYSw5JagMfI7EzY0T8kVYikiyR1kBl2uk/SdoCI+DmZHsrTkraRWaa7pBzDoM/2wntmZpbLPQczMxvEwcHMzAZxcDAzs0EcHMzMbBAHBzMzG8TBwczMBnFwMDOzQRwczMxskP8PKwcgNZwjtxwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "gruDemo = GRUDemo(input_dim,hidden_size,num_layers).cuda()\n",
    "optimizer = torch.optim.Adam(gruDemo.parameters(), lr=lr)\n",
    "loss_func = nn.MSELoss()\n",
    "\n",
    "c_state = torch.zeros(num_layers*1,1,hidden_size).cuda()\n",
    "for step in range(epochs):\n",
    "    start, end = step * np.pi, (step + 1) * np.pi  # 时间跨度\n",
    "    # 使用Sin函数预测Cos函数\n",
    "    steps = np.linspace(start, end, bins, dtype=np.float32, endpoint=False)\n",
    "    x_np = np.sin(steps)\n",
    "    y_np = np.cos(steps)\n",
    "    x = torch.from_numpy(x_np).unsqueeze(1).unsqueeze(2).cuda()#【100，1,1】尺寸大小为(time_step, batch, input_size)\n",
    "    y = torch.from_numpy(y_np).unsqueeze(1).unsqueeze(2).cuda()#【100，1,1】\n",
    "    prediction, c_state = gruDemo(x, c_state)  # RNN输出（预测结果，隐藏状态）\n",
    "    #将每一次输出的中间状态传递下去(不带梯度)\n",
    "    c_state = c_state.detach()  \n",
    "    loss = loss_func(prediction, y)\n",
    "    optimizer.zero_grad()\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    if(step%100==0):\n",
    "        print(\"loss:{:.8f}\".format(loss))\n",
    "plt.scatter(steps,y_np,marker=\"^\")\n",
    "plt.scatter(steps, prediction.cpu().data.numpy().flatten(),marker=\".\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = nn.Linear(10,20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 318,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([20, 10])"
      ]
     },
     "execution_count": 318,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.weight.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
